rocprofiler-sdk-tool library intermediate binary output (#734)

* Support for binary temporary files * clang formatting * formating ring buffer.hpp * Update source/lib/common/container/ring_buffer.hpp Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fixing bugs * fix loop range * Fix for v3 test failures * bug fix * fix bug * fix memory leaks * destructing agent_info * Update CMakeLists.txt * clang-tidy fixes * Fix data race on destructor of rocprofiler_agent_t map in rocprofiler-sdk-tool library * Create lib/rocproifler-sdk-tool/tmp_file.* - move tmp_file class into separate header/implementation * Agent Info CSV in rocprofiler-sdk-tool - update tests to use agent_info.csv instead of rocminfo * Update lib/rocprofiler-sdk-tool/tool.cpp - use logical_node_id instead of node_id * Adding stats file * Adding tests for stats * Update scratch memory support - convert scratch memory support to use binary output * Tool Update: scratch memory stats + extended statistics - replace generate_*_csv with generate_csv overloads - added generate_csv for scratch memory - enable stats for scratch memory - replace ROCPROF_*_STATS env variables with ROCPROF_STATS env variable * rocprofv3 update - simple --stats option - add scratch memory trace to --sys-trace * Update tests/rocprofv3/tracing-hip-in-libraries - extend validate.py to test stats data - fix conftest.py for memory_copy_stats_data * Code coverage fixes - invoke __gcov_dump to ensure that code coverage is flushed after finalization --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Ammar ELWazir <ammar.elwazir@amd.com> Co-authored-by: Jonathan R. Madsen <jonathanrmadsen@gmail.com>
2024-04-09 05:25:28 -05:00
@@ -31,12 +31,13 @@ def test_validate_counter_collection_pmc2(input_dir: pd.DataFrame):
            file_path = os.path.join(subdirectory_path, file_name)
            assert os.path.isfile(file_path), f"{file_path} is not a file."

-            with open(file_path, "r") as file:
-                df = pd.read_csv(file)
-                # check if kernel-name is present
-                assert len(df["Kernel_Name"]) > 0
-                # check if counter value is positive
-                assert len(df["Counter_Value"]) > 0
+            if "agent_info.csv" not in file_path:
+                with open(file_path, "r") as file:
+                    df = pd.read_csv(file)
+                    # check if kernel-name is present
+                    assert len(df["Kernel_Name"]) > 0
+                    # check if counter value is positive
+                    assert len(df["Counter_Value"]) > 0


 if __name__ == "__main__":
@@ -15,7 +15,7 @@ add_test(
    COMMAND
        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hip-runtime-trace
        --hip-compiler-trace --hsa-core-trace --hsa-amd-trace --hsa-image-trace
-        --hsa-finalizer-trace --kernel-trace --memory-copy-trace -d
+        --hsa-finalizer-trace --kernel-trace --memory-copy-trace --stats -d
        ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out $<TARGET_FILE:hip-in-libraries>)

 string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
@@ -51,13 +51,26 @@ add_test(
        --kernel-input
        ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_kernel_trace.csv
        --memory-copy-input
-        ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_trace.csv)
+        ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_trace.csv
+        --agent-input
+        ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_agent_info.csv
+        --kernel-stats
+        ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_kernel_stats.csv
+        --hip-stats ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hip_stats.csv
+        --hsa-stats ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hsa_stats.csv
+        --memory-copy-stats
+        ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_stats.csv)

 set(VALIDATION_FILES
    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_trace.csv
    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hsa_api_trace.csv
    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hip_api_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_kernel_trace.csv)
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_kernel_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_agent_info.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_kernel_stats.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hip_stats.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_hsa_stats.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_stats.csv)

 set_tests_properties(
    rocprofv3-test-trace-hip-in-libraries-validate
@@ -6,6 +6,11 @@ import pytest


 def pytest_addoption(parser):
+    parser.addoption(
+        "--agent-input",
+        action="store",
+        help="Path to agent info CSV file.",
+    )
    parser.addoption(
        "--hsa-input",
        action="store",
@@ -31,6 +36,38 @@ def pytest_addoption(parser):
        action="store",
        help="Path to HIP runtime and compiler API tracing CSV file.",
    )
+    parser.addoption(
+        "--hip-stats",
+        action="store",
+        help="Path to HIP stats CSV file.",
+    )
+    parser.addoption(
+        "--hsa-stats",
+        action="store",
+        help="Path to HSA stats CSV file.",
+    )
+    parser.addoption(
+        "--kernel-stats",
+        action="store",
+        help="Path to kernel stats CSV file.",
+    )
+    parser.addoption(
+        "--memory-copy-stats",
+        action="store",
+        help="Path to memory copy stats CSV file.",
+    )
+
+
+@pytest.fixture
+def agent_info_input_data(request):
+    filename = request.config.getoption("--agent-input")
+    data = []
+    with open(filename, "r") as inp:
+        reader = csv.DictReader(inp)
+        for row in reader:
+            data.append(row)
+
+    return data


@pytest.fixture
@@ -92,3 +129,55 @@ def hip_input_data(request):
                data.append(row)

    return data
+
+
+@pytest.fixture
+def hip_stats_data(request):
+    filename = request.config.getoption("--hip-stats")
+    data = []
+    if os.path.exists(filename):
+        with open(filename, "r") as inp:
+            reader = csv.DictReader(inp)
+            for row in reader:
+                data.append(row)
+
+    return data
+
+
+@pytest.fixture
+def hsa_stats_data(request):
+    filename = request.config.getoption("--hsa-stats")
+    data = []
+    if os.path.exists(filename):
+        with open(filename, "r") as inp:
+            reader = csv.DictReader(inp)
+            for row in reader:
+                data.append(row)
+
+    return data
+
+
+@pytest.fixture
+def kernel_stats_data(request):
+    filename = request.config.getoption("--kernel-stats")
+    data = []
+    if os.path.exists(filename):
+        with open(filename, "r") as inp:
+            reader = csv.DictReader(inp)
+            for row in reader:
+                data.append(row)
+
+    return data
+
+
+@pytest.fixture
+def memory_copy_stats_data(request):
+    filename = request.config.getoption("--memory-copy-stats")
+    data = []
+    if os.path.exists(filename):
+        with open(filename, "r") as inp:
+            reader = csv.DictReader(inp)
+            for row in reader:
+                data.append(row)
+
+    return data
@@ -3,7 +3,6 @@
 import re
 import sys
 import pytest
-import subprocess


 class dim3(object):
@@ -16,7 +15,29 @@ class dim3(object):
        return (self.x, self.y, self.z)


-def test_api_trace(hsa_input_data, hip_input_data):
+def validate_average(row):
+    avg_ns = float(row["AverageNs"])
+    tot_ns = int(row["TotalDurationNs"])
+    num_cnt = float(row["Calls"])
+    assert abs(avg_ns - (tot_ns / num_cnt)) < 1.0e-3, f"{row}"
+
+
+def validate_stats(row):
+    min_v = int(row["MinNs"])
+    max_v = int(row["MaxNs"])
+    avg_v = float(row["AverageNs"])
+    cnt_v = int(row["Calls"])
+    stddev_v = float(row["StdDev"])
+
+    assert min_v > 0, f"{row}"
+    assert max_v > 0, f"{row}"
+    assert min_v < max_v if cnt_v > 1 else min_v == max_v, f"{row}"
+    assert min_v < avg_v if cnt_v > 1 else min_v == int(avg_v), f"{row}"
+    assert max_v > avg_v if cnt_v > 1 else max_v == int(avg_v), f"{row}"
+    assert stddev_v > 0.0 if cnt_v > 1 else int(stddev_v) == 0, f"{row}"
+
+
+def test_api_trace(hsa_input_data, hip_input_data, hip_stats_data):
    functions = []
    correlation_ids = []
    for row in hsa_input_data:
@@ -77,8 +98,15 @@ def test_api_trace(hsa_input_data, hip_input_data):
        ):
            assert itr in functions

+    for row in hip_stats_data:
+        assert int(row["TotalDurationNs"]) > 0
+        assert int(row["Calls"]) > 0
+        validate_average(row)
+        assert float(row["Percentage"]) > 0.0
+        validate_stats(row)

-def test_kernel_trace(kernel_input_data):
+
+def test_kernel_trace(kernel_input_data, kernel_stats_data):
    valid_kernel_names = sorted(
        [
            "(anonymous namespace)::transpose(int const*, int*, int, int)",
@@ -125,27 +153,42 @@ def test_kernel_trace(kernel_input_data):
    kernels = sorted(list(set(kernels)))
    assert kernels == valid_kernel_names

+    for row in kernel_stats_data:
+        assert int(row["TotalDurationNs"]) > 0
+        assert int(row["Calls"]) > 0
+        validate_average(row)
+        assert float(row["Percentage"]) > 0.0
+        validate_stats(row)
+
+
+def test_memory_copy_trace(
+    agent_info_input_data,
+    memory_copy_input_data,
+    hsa_input_data,
+    hsa_stats_data,
+    memory_copy_stats_data,
+):
+    def get_agent(node_id):
+        for row in agent_info_input_data:
+            if row["Logical_Node_Id"] == node_id:
+                return row
+        return None

-def test_memory_copy_trace(memory_copy_input_data, hsa_input_data):
    for row in memory_copy_input_data:
        assert row["Kind"] == "MEMORY_COPY"
        assert row["Direction"] in ("HOST_TO_DEVICE", "DEVICE_TO_HOST")
+
+        src_agent = get_agent(row["Source_Agent_Id"])
+        dst_agent = get_agent(row["Destination_Agent_Id"])
+        assert src_agent is not None and dst_agent is not None, f"{agent_info_input_data}"
+
        if row["Direction"] == "HOST_TO_DEVICE":
-            output = subprocess.check_output(
-                'rocminfo | grep "Node: *'
-                + row["Source_Agent_Id"]
-                + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
-                shell=True,
-            )
-            assert int(str(output).find("CPU")) >= 0
+            assert src_agent["Agent_Type"] == "CPU"
+            assert dst_agent["Agent_Type"] == "GPU"
        elif row["Direction"] == "DEVICE_TO_HOST":
-            output = subprocess.check_output(
-                'rocminfo | grep "Node: *'
-                + row["Destination_Agent_Id"]
-                + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
-                shell=True,
-            )
-            assert int(str(output).find("CPU")) >= 0
+            assert src_agent["Agent_Type"] == "GPU"
+            assert dst_agent["Agent_Type"] == "CPU"
+
        assert int(row["Correlation_Id"]) > 0
        assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])

@@ -155,6 +198,20 @@ def test_memory_copy_trace(memory_copy_input_data, hsa_input_data):
            valid_length += 1
    assert len(memory_copy_input_data) == valid_length

+    for row in hsa_stats_data:
+        assert int(row["TotalDurationNs"]) > 0
+        assert int(row["Calls"]) > 0
+        validate_average(row)
+        assert float(row["Percentage"]) > 0.0
+        validate_stats(row)
+
+    for row in memory_copy_stats_data:
+        assert int(row["TotalDurationNs"]) > 0
+        assert int(row["Calls"]) > 0
+        validate_average(row)
+        assert float(row["Percentage"]) > 0.0
+        validate_stats(row)
+

 if __name__ == "__main__":
    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
@@ -31,10 +31,11 @@ def test_validate_counter_collection_plus_tracing(input_dir: pd.DataFrame):
            file_path = os.path.join(subdirectory_path, file_name)
            assert os.path.isfile(file_path), f"{file_path} is not a file."

-            with open(file_path, "r") as file:
-                df = pd.read_csv(file)
-                # check if either kernel-name/FUNCTION is present
-                assert "Kernel_Name" in df.columns or "Function" in df.columns
+            if "agent_info.csv" not in file_path:
+                with open(file_path, "r") as file:
+                    df = pd.read_csv(file)
+                    # check if either kernel-name/FUNCTION is present
+                    assert "Kernel_Name" in df.columns or "Function" in df.columns


 if __name__ == "__main__":
@@ -56,13 +56,16 @@ add_test(
        --memory-copy-input
        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_memory_copy_trace.csv
        --marker-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_marker_api_trace.csv)
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_marker_api_trace.csv
+        --agent-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_agent_info.csv)

 set(VALIDATION_FILES
    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_memory_copy_trace.csv
    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_hsa_api_trace.csv
    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_kernel_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_marker_api_trace.csv)
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_marker_api_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-trace/out_agent_info.csv)

 set_tests_properties(
    rocprofv3-test-trace-validate
@@ -111,13 +114,16 @@ add_test(
        --memory-copy-input
        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_memory_copy_trace.csv
        --marker-input
-        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_marker_api_trace.csv)
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_marker_api_trace.csv
+        --agent-input
+        ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_agent_info.csv)

 set(SYS_VALIDATION_FILES
    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_memory_copy_trace.csv
    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_hsa_api_trace.csv
    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_kernel_trace.csv
-    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_marker_api_trace.csv)
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_marker_api_trace.csv
+    ${CMAKE_CURRENT_BINARY_DIR}/simple-transpose-systrace/out_agent_info.csv)

 set_tests_properties(
    rocprofv3-test-systrace-validate
@@ -5,6 +5,11 @@ import pytest


 def pytest_addoption(parser):
+    parser.addoption(
+        "--agent-input",
+        action="store",
+        help="Path to agent info CSV file.",
+    )
    parser.addoption(
        "--hsa-input",
        action="store",
@@ -32,6 +37,18 @@ def pytest_addoption(parser):
    )


+@pytest.fixture
+def agent_info_input_data(request):
+    filename = request.config.getoption("--agent-input")
+    data = []
+    with open(filename, "r") as inp:
+        reader = csv.DictReader(inp)
+        for row in reader:
+            data.append(row)
+
+    return data
+
+
@pytest.fixture
 def hsa_input_data(request):
    filename = request.config.getoption("--hsa-input")
@@ -1,10 +1,27 @@
 #!/usr/bin/env python3

 import sys
-import subprocess
 import pytest


+def test_agent_info(agent_info_input_data):
+    logical_node_id = max([int(itr["Logical_Node_Id"]) for itr in agent_info_input_data])
+
+    assert logical_node_id + 1 == len(agent_info_input_data)
+
+    for row in agent_info_input_data:
+        agent_type = row["Agent_Type"]
+        assert agent_type in ("CPU", "GPU")
+        if agent_type == "CPU":
+            assert int(row["Cpu_Cores_Count"]) > 0
+            assert int(row["Simd_Count"]) == 0
+            assert int(row["Max_Waves_Per_Simd"]) == 0
+        else:
+            assert int(row["Cpu_Cores_Count"]) == 0
+            assert int(row["Simd_Count"]) > 0
+            assert int(row["Max_Waves_Per_Simd"]) > 0
+
+
 def test_hsa_api_trace(hsa_input_data):
    functions = []
    correlation_ids = []
@@ -60,49 +77,37 @@ def test_kernel_trace(kernel_input_data):
        assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])


-def test_memory_copy_trace(memory_copy_input_data):
+def test_memory_copy_trace(agent_info_input_data, memory_copy_input_data):
+
+    def get_agent(node_id):
+        for row in agent_info_input_data:
+            if row["Logical_Node_Id"] == node_id:
+                return row
+        return None
+
    for row in memory_copy_input_data:
        assert row["Kind"] == "MEMORY_COPY"

    assert len(memory_copy_input_data) == 2

-    row = memory_copy_input_data[0]
-    assert row["Direction"] == "HOST_TO_DEVICE"
-    output = subprocess.check_output(
-        'rocminfo | grep "Node: *'
-        + row["Source_Agent_Id"]
-        + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
-        shell=True,
-    )
-    assert int(str(output).find("CPU")) >= 0
-    output = subprocess.check_output(
-        'rocminfo | grep "Node: *'
-        + row["Destination_Agent_Id"]
-        + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
-        shell=True,
-    )
-    assert int(str(output).find("GPU")) >= 0
-    assert int(row["Correlation_Id"]) > 0
-    assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])
+    def test_row(idx, direction):
+        assert direction in ("HOST_TO_DEVICE", "DEVICE_TO_HOST")
+        row = memory_copy_input_data[idx]
+        assert row["Direction"] == direction
+        src_agent = get_agent(row["Source_Agent_Id"])
+        dst_agent = get_agent(row["Destination_Agent_Id"])
+        assert src_agent is not None and dst_agent is not None, f"{agent_info_input_data}"
+        if direction == "HOST_TO_DEVICE":
+            assert src_agent["Agent_Type"] == "CPU"
+            assert dst_agent["Agent_Type"] == "GPU"
+        else:
+            assert src_agent["Agent_Type"] == "GPU"
+            assert dst_agent["Agent_Type"] == "CPU"
+        assert int(row["Correlation_Id"]) > 0
+        assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])

-    row = memory_copy_input_data[1]
-    assert row["Direction"] == "DEVICE_TO_HOST"
-    output = subprocess.check_output(
-        'rocminfo | grep "Node: *'
-        + row["Source_Agent_Id"]
-        + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
-        shell=True,
-    )
-    assert int(str(output).find("GPU")) >= 0
-    output = subprocess.check_output(
-        'rocminfo | grep "Node: *'
-        + row["Destination_Agent_Id"]
-        + '" -A 1 | grep "Device Type" | sed \'s/.*: *//\'',
-        shell=True,
-    )
-    assert int(str(output).find("CPU")) >= 0
-    assert int(row["Correlation_Id"]) > 0
-    assert int(row["End_Timestamp"]) >= int(row["Start_Timestamp"])
+    test_row(0, "HOST_TO_DEVICE")
+    test_row(1, "DEVICE_TO_HOST")


 def test_marker_api_trace(marker_input_data):