rocprofv3 kernel renaming support + misc rocprofv3 updates (#992)

* Increase rocprofv3 tool buffer size

- 32 pages instead of 1 page

* Improve rocprofv3 perfetto track labels

* Preliminary kernel renaming support + misc rocprofv3 updates

- add rocprofv3 option --kernel-rename
- add rocprofv3 options for perfetto settings (buffer size, etc.)
- add CSV columns for kernel trace
  - Thread_Id
  - Dispatch_Id
- add CSV column for counter_collection
 - Kernel_Id

tests/bin/transpose/transpose.cpp

@@ -206,8 +206,8 @@ run(int rank, int tid, int devid, int argc, char** argv)
     int* in  = nullptr;
     int* out = nullptr;
 
-    HIP_API_CALL(hipMalloc(&in, size));
-    HIP_API_CALL(hipMalloc(&out, size));
+    HIP_API_CALL(hipMallocAsync(&in, size, stream));
+    HIP_API_CALL(hipMallocAsync(&out, size, stream));
     HIP_API_CALL(hipMemsetAsync(in, 0, size, stream));
     HIP_API_CALL(hipMemsetAsync(out, 0, size, stream));
     HIP_API_CALL(hipMemcpyAsync(in, inp_matrix, size, hipMemcpyHostToDevice, stream));
@@ -238,13 +238,15 @@ run(int rank, int tid, int devid, int argc, char** argv)
     print_lock.unlock();
 
     HIP_API_CALL(hipStreamSynchronize(stream));
-    HIP_API_CALL(hipStreamDestroy(stream));
 
     // cpu_transpose(matrix, out_matrix, M, N);
     verify(inp_matrix, out_matrix, M, N);
 
-    HIP_API_CALL(hipFree(in));
-    HIP_API_CALL(hipFree(out));
+    HIP_API_CALL(hipFreeAsync(in, stream));
+    HIP_API_CALL(hipFreeAsync(out, stream));
+
+    HIP_API_CALL(hipStreamSynchronize(stream));
+    HIP_API_CALL(hipStreamDestroy(stream));
 
     delete[] inp_matrix;
     delete[] out_matrix;

tests/rocprofv3/CMakeLists.txt

@@ -28,3 +28,4 @@ add_subdirectory(tracing-plus-counter-collection)
 add_subdirectory(tracing-hip-in-libraries)
 add_subdirectory(counter-collection)
 add_subdirectory(hsa-queue-dependency)
+add_subdirectory(kernel-rename)

tests/rocprofv3/kernel-rename/CMakeLists.txt

@@ -0,0 +1,114 @@
+#
+# rocprofv3 tool tests for kernel renaming
+#
+cmake_minimum_required(VERSION 3.21.0 FATAL_ERROR)
+
+project(
+    rocprofiler-tests-rocprofv3-kernel-rename
+    LANGUAGES CXX
+    VERSION 0.0.0)
+
+find_package(rocprofiler-sdk REQUIRED)
+
+if(ROCPROFILER_MEMCHECK STREQUAL "LeakSanitizer")
+    set(LOG_LEVEL "warning") # info produces memory leak
+else()
+    set(LOG_LEVEL "info")
+endif()
+
+string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
+               "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")
+
+set(kernel-rename-env "${PRELOAD_ENV}")
+
+rocprofiler_configure_pytest_files(CONFIG pytest.ini input-kernel-rename.yml
+                                   COPY validate.py conftest.py)
+
+##########################################################################################
+#
+#   Command line input
+#
+##########################################################################################
+
+add_test(
+    NAME rocprofv3-test-kernel-rename-cmd-line-execute
+    COMMAND
+        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -M --sys-trace no --hsa-trace=0
+        --hsa-core-trace=1 --hip-compiler-trace False --hip-runtime-trace --kernel-trace
+        --memory-copy-trace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace-cmd-line -o out
+        --output-format pftrace json --log-level env --kernel-rename
+        --perfetto-shmem-size-hint 128 --perfetto-buffer-size 2048000
+        --perfetto-buffer-fill-policy ring_buffer --perfetto-backend inprocess --
+        $<TARGET_FILE:transpose>)
+
+set_tests_properties(
+    rocprofv3-test-kernel-rename-cmd-line-execute
+    PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${kernel-rename-env}"
+               FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
+
+add_test(
+    NAME rocprofv3-test-kernel-rename-cmd-line-validate
+    COMMAND
+        ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
+        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.json
+        --pftrace-input
+        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.pftrace)
+
+set(VALIDATION_FILES
+    ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.pftrace
+    ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.json)
+
+set_tests_properties(
+    rocprofv3-test-kernel-rename-cmd-line-validate
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests"
+               DEPENDS
+               "rocprofv3-test-kernel-rename-execute"
+               FAIL_REGULAR_EXPRESSION
+               "AssertionError"
+               ATTACHED_FILES_ON_FAIL
+               "${VALIDATION_FILES}")
+
+##########################################################################################
+#
+#   YAML input
+#
+##########################################################################################
+
+add_test(
+    NAME rocprofv3-test-kernel-rename-inp-yaml-execute
+    COMMAND
+        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -i
+        ${CMAKE_CURRENT_BINARY_DIR}/input-kernel-rename.yml -- $<TARGET_FILE:transpose>)
+
+set_tests_properties(
+    rocprofv3-test-kernel-rename-inp-yaml-execute
+    PROPERTIES TIMEOUT 45 LABELS "integration-tests" ENVIRONMENT "${kernel-rename-env}"
+               FAIL_REGULAR_EXPRESSION "${ROCPROFILER_DEFAULT_FAIL_REGEX}")
+
+add_test(
+    NAME rocprofv3-test-kernel-rename-inp-yaml-validate
+    COMMAND
+        ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
+        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.json
+        --pftrace-input
+        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.pftrace)
+
+set(VALIDATION_FILES
+    ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.pftrace
+    ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.json)
+
+set_tests_properties(
+    rocprofv3-test-kernel-rename-inp-yaml-validate
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests"
+               DEPENDS
+               "rocprofv3-test-kernel-rename-inp-yaml-execute"
+               FAIL_REGULAR_EXPRESSION
+               "AssertionError"
+               ATTACHED_FILES_ON_FAIL
+               "${VALIDATION_FILES}")

tests/rocprofv3/kernel-rename/conftest.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import pytest
+import json
+
+from rocprofiler_sdk.pytest_utils.dotdict import dotdict
+from rocprofiler_sdk.pytest_utils import collapse_dict_list
+from rocprofiler_sdk.pytest_utils.perfetto_reader import PerfettoReader
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--json-input",
+        action="store",
+        help="Path to JSON file.",
+    )
+    parser.addoption(
+        "--pftrace-input",
+        action="store",
+        help="Path to Perfetto trace file.",
+    )
+
+
+@pytest.fixture
+def json_data(request):
+    filename = request.config.getoption("--json-input")
+    with open(filename, "r") as inp:
+        return dotdict(collapse_dict_list(json.load(inp)))
+
+
+@pytest.fixture
+def pftrace_data(request):
+    filename = request.config.getoption("--pftrace-input")
+    return PerfettoReader(filename).read()[0]

tests/rocprofv3/kernel-rename/input-kernel-rename.yml

@@ -0,0 +1,20 @@
+jobs:
+  - mangled_kernels: True
+    sys_trace: False
+    hsa_trace: False
+    hsa_core_trace: True
+    hsa_amd_trace: False
+    hip_compiler_trace: False
+    hip_runtime_trace: True
+    kernel_trace: True
+    memory_copy_trace: True
+    marker_trace: False
+    output_directory: "@CMAKE_CURRENT_BINARY_DIR@/%argt%-trace-inp-yaml"
+    output_file: out
+    output_format: [pftrace, json]
+    log_level: env
+    kernel_rename: True
+    perfetto_shmem_size_hint: 128
+    perfetto_buffer_size: 2048000
+    perfetto_buffer_fill_policy: ring_buffer
+    perfetto_backend: inprocess

tests/rocprofv3/kernel-rename/pytest.ini

@@ -0,0 +1,5 @@
+
+[pytest]
+addopts = --durations=20 -rA -s -vv
+testpaths = validate.py
+pythonpath = @ROCPROFILER_SDK_TESTS_BINARY_DIR@/pytest-packages

tests/rocprofv3/kernel-rename/validate.py

@@ -0,0 +1,148 @@
+#!/usr/bin/env python3
+
+import sys
+import pytest
+
+
+def test_hsa_api_trace(json_data):
+    data = json_data["rocprofiler-sdk-tool"]
+
+    def get_operation_name(kind_id, op_id):
+        return data["strings"]["buffer_records"][kind_id]["operations"][op_id]
+
+    def get_kind_name(kind_id):
+        return data["strings"]["buffer_records"][kind_id]["kind"]
+
+    valid_domain_names = ("HSA_CORE_API",)
+
+    hsa_api_data = data["buffer_records"]["hsa_api"]
+
+    functions = []
+    for api in hsa_api_data:
+        kind = get_kind_name(api["kind"])
+        assert kind in valid_domain_names
+        assert api["end_timestamp"] >= api["start_timestamp"]
+        functions.append(get_operation_name(api["kind"], api["operation"]))
+
+    functions = list(set(functions))
+    assert "hsa_amd_memory_async_copy_on_engine" not in functions
+    assert "hsa_signal_destroy" in functions
+
+
+def test_hip_api_trace(json_data):
+    data = json_data["rocprofiler-sdk-tool"]
+
+    def get_operation_name(kind_id, op_id):
+        return data["strings"]["buffer_records"][kind_id]["operations"][op_id]
+
+    def get_kind_name(kind_id):
+        return data["strings"]["buffer_records"][kind_id]["kind"]
+
+    valid_domain_names = ("HIP_RUNTIME_API",)
+
+    hip_api_data = data["buffer_records"]["hip_api"]
+
+    functions = []
+    for api in hip_api_data:
+        kind = get_kind_name(api["kind"])
+        assert kind in valid_domain_names
+        assert api["end_timestamp"] >= api["start_timestamp"]
+        functions.append(get_operation_name(api["kind"], api["operation"]))
+
+    functions = list(set(functions))
+    for itr in (
+        "__hipPushCallConfiguration",
+        "__hipPopCallConfiguration",
+        "__hipRegisterFatBinary",
+        "__hipRegisterFunction",
+    ):
+        assert itr not in functions, f"{itr}"
+
+    for itr in (
+        "hipMallocAsync",
+        "hipMemcpyAsync",
+        "hipMemsetAsync",
+        "hipFreeAsync",
+        "hipLaunchKernel",
+    ):
+        assert itr in functions, f"{itr}"
+
+
+def test_kernel_trace(json_data):
+    data = json_data["rocprofiler-sdk-tool"]
+
+    def get_kernel_name(kernel_id):
+        return data["kernel_symbols"][kernel_id]["formatted_kernel_name"]
+
+    def get_kernel_rename(corr_id):
+        for itr in data.strings.correlation_id.external:
+            if itr.key == corr_id:
+                return itr.value
+        return None
+
+    def get_kind_name(kind_id):
+        return data["strings"]["buffer_records"][kind_id]["kind"]
+
+    valid_kernel_names = ("run",)
+    kernel_dispatch_data = data["buffer_records"]["kernel_dispatch"]
+    for dispatch in kernel_dispatch_data:
+        assert get_kind_name(dispatch["kind"]) == "KERNEL_DISPATCH"
+        assert dispatch["correlation_id"]["internal"] > 0
+        assert dispatch["correlation_id"]["external"] > 0
+
+        dispatch_info = dispatch["dispatch_info"]
+        assert dispatch_info["agent_id"]["handle"] > 0
+        assert dispatch_info["queue_id"]["handle"] > 0
+        assert dispatch_info["kernel_id"] > 0
+        assert dispatch["end_timestamp"] >= dispatch["start_timestamp"]
+
+        kernel_name = get_kernel_name(dispatch_info["kernel_id"])
+        assert kernel_name not in valid_kernel_names
+
+        external_corr_id = dispatch["correlation_id"]["external"]
+        assert external_corr_id > 0
+
+        kernel_rename = get_kernel_rename(external_corr_id)
+        assert kernel_rename is not None, f"{dispatch}"
+        assert kernel_rename in valid_kernel_names
+
+
+def test_memory_copy_json_trace(json_data):
+    data = json_data["rocprofiler-sdk-tool"]
+
+    buffer_records = data["buffer_records"]
+    agent_data = data["agents"]
+    memory_copy_data = buffer_records["memory_copy"]
+
+    def get_kind_name(kind_id):
+        return data["strings"]["buffer_records"][kind_id]["kind"]
+
+    def get_agent(node_id):
+        for agent in agent_data:
+            if agent["id"]["handle"] == node_id["handle"]:
+                return agent
+        return None
+
+    assert len(memory_copy_data) == 12
+
+    for row in memory_copy_data:
+        src_agent = get_agent(row["src_agent_id"])
+        dst_agent = get_agent(row["dst_agent_id"])
+        assert get_kind_name(row["kind"]) == "MEMORY_COPY"
+        assert src_agent is not None, f"{row}"
+        assert dst_agent is not None, f"{row}"
+        assert row["correlation_id"]["internal"] > 0
+        assert row["end_timestamp"] >= row["start_timestamp"]
+
+
+def test_perfetto_data(pftrace_data, json_data):
+    import rocprofiler_sdk.tests.rocprofv3 as rocprofv3
+
+    rocprofv3.test_perfetto_data(
+        pftrace_data, json_data, ("hip", "hsa", "kernel", "memory_copy")
+    )
+
+
+if __name__ == "__main__":
+    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
+    sys.exit(exit_code)