rocprofv3 OTF2 Output Support (#995)

* CMake support for OTF2 library * Preliminary OTF2 generation implementation * Completed OTF2 Support - HSA API - HIP API - Marker API - Async Memory Copies - Kernel Dispatch * Update lib/rocprofiler-sdk-tool/generateOTF2.cpp - fix location type for dispatches * Testing for OTF2 output * Add OTF2 to requirements.txt * Update lib/rocprofiler-sdk-tool/generateOTF2.cpp - fix getting kernel name * OTF2 testing with rocprofv3/tracing-hip-in-libraries * Format external/otf2/CMakeLists.txt * Update external/otf2/CMakeLists.txt - guard CMP0135 for cmake < 3.24 * Update lib/rocprofiler-sdk-tool/generateOTF2.cpp - fix duplicate string ref issue * Update lib/rocprofiler-sdk-tool/generateOTF2.cpp - fix header includes * Update CI workflow - sudo install pypi requirements for core-rpm for $HOME/.local installs * Update pytest_utils/otf2_reader.py - modifications for reading trace * Update pytest_utils/otf2_reader.py - misc cleanup * Update CI workflow - fix installer artifact naming * Update pytest_utils/otf2_reader.py - handle slightly overlapping kernel timestamps for MI300 * OTF2 attributes for category * Testing with OTF2Reader category attributes * Fix memory leak in OTF2 generation - leaking OTF2_AttributeList
2024-07-30 19:57:19 -05:00
1 parent 3cd080e63b
commit 16d535ef48
@@ -5,7 +5,7 @@
 set(PACKAGE_OUTPUT_DIR
    ${ROCPROFILER_SDK_TESTS_BINARY_DIR}/pytest-packages/rocprofiler_sdk)

-set(PYTEST_UTILS_PYTHON_SOURCES __init__.py dotdict.py perfetto_reader.py)
+set(PYTEST_UTILS_PYTHON_SOURCES __init__.py dotdict.py perfetto_reader.py otf2_reader.py)
 set(TESTS_PYTHON_SOURCES __init__.py rocprofv3.py)

 foreach(_FILE ${PYTEST_UTILS_PYTHON_SOURCES})
@@ -0,0 +1,201 @@
+# MIT License
+#
+# Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from __future__ import absolute_import
+
+import otf2
+import pandas as pd
+
+from otf2.events import Enter, Leave
+
+
+class Region(object):
+    """ """
+
+    def __init__(self, _enter, _leave, _depth, _location=None):
+        if _enter.region != _leave.region:
+            _location_info = f"\n{_location}" if _location else ""
+            raise ValueError(
+                f"enter region != leave region :: '{_enter}' != '{_leave}'{_location_info}"
+            )
+
+        if _depth < 0:
+            _location_info = f". location: '{_location}'" if _location else ""
+            raise ValueError(
+                f"negative depth ({_depth})! enter: '{_enter}'. leave: '{_leave}'{_location_info}"
+            )
+
+        self.region = _enter.region
+        self.depth = _depth
+        self.name = _enter.region.name
+        self.attributes = [
+            itr for itr in [_enter.attributes, _leave.attributes] if itr is not None
+        ]
+        self.enter_nsec = _enter.time
+        self.leave_nsec = _leave.time
+        self.delta_nsec = _leave.time - _enter.time
+        if self.delta_nsec < 0:
+            raise ValueError(
+                f"negative timestamp delta :: '{_enter.time}' > '{_leave.time}'"
+            )
+
+        for itr in self.attributes:
+            for key, val in itr.items():
+                _key = f"{key.name}"
+                if not hasattr(self, _key):
+                    setattr(self, _key, val)
+
+        if not hasattr(self, "category"):
+            self.category = "unk"
+
+    def __str__(self):
+        return f"{self.name:<35} :: {self.delta_nsec} nsec"
+
+
+class OTF2Reader:
+    """Read in perfetto protobuf output"""
+
+    def __init__(self, filename):
+        self.filename = filename if isinstance(filename, (list, tuple)) else [filename]
+
+    def read(self):
+
+        def _read_trace(trace_name):
+            trace = otf2.reader.Reader(trace_name)
+            # print(f"Read {len(trace.definitions.strings)} string definitions")
+            # for string in trace.definitions.strings:
+            #     print(f"String definition with value '{string}' in trace.")
+            # print("Read {} events".format(len(trace.events)))
+
+            events = [[loc, evt] for loc, evt in trace.events]
+            locations = [itr for itr in trace.definitions.locations]
+            location_groups = [itr for itr in trace.definitions.location_groups]
+            system_tree_nodes = [itr for itr in trace.definitions.system_tree_nodes]
+
+            call_stack = {}
+            partial_call_stack = {}
+
+            for itr in system_tree_nodes:
+                call_stack[itr] = {}
+                partial_call_stack[itr] = {}
+
+            for itr in location_groups:
+                call_stack[itr.system_tree_parent][itr] = {}
+                partial_call_stack[itr.system_tree_parent][itr] = {}
+
+            for itr in locations:
+                call_stack[itr.group.system_tree_parent][itr.group][itr] = []
+                partial_call_stack[itr.group.system_tree_parent][itr.group][itr] = []
+
+            for location, event in events:
+                _stree = location.group.system_tree_parent
+                _group = location.group
+                _partial = partial_call_stack[_stree][_group][location]
+                if isinstance(event, Enter):
+                    # expected length
+                    _elen = len(_partial) + 1
+                    _partial.append(event)
+                elif isinstance(event, Leave):
+                    # expected length
+                    _elen = len(_partial) - 1
+                    _depth = len(_partial)
+                    _leave = event
+
+                    # it appears that on MI300, the end of A may exceed the
+                    # begin of B kernels very slightly (i.e. overlap in same
+                    # stream/queue). This leads to slightly out of order
+                    # Enter/Leave regions and thus we need to occasionally
+                    # search further back in the callstack to find the correct
+                    # Enter region
+                    _enter = _partial[-1]
+                    if _enter.region == _leave.region:
+                        _partial.pop()
+                    else:
+                        for ridx, ritr in enumerate(reversed(_partial)):
+                            if ritr.region == _leave.region:
+                                _enter = _partial.pop(len(_partial) - ridx - 1)
+                                break
+
+                    # below is what is expected on non-MI300
+                    # _enter = _partial.pop()
+
+                    # add the region
+                    call_stack[_stree][_group][location].append(
+                        Region(_enter, _leave, _depth - 1, location)
+                    )
+
+                # modified length
+                _mlen = len(partial_call_stack[_stree][_group][location])
+                # if modified length != expected length
+                if _mlen != _elen:
+                    raise RuntimeError(
+                        f"Modified length ({_mlen}) != Expected length({_elen}) for {event} at {location}"
+                    )
+
+            data = {
+                "system_tree_node": [],
+                "location_group": [],
+                "location": [],
+                "region": [],
+                "attributes": [],
+                "depth": [],
+                "name": [],
+                "category": [],
+                "start_ts": [],
+                "end_ts": [],
+            }
+
+            for tree, lgitr in call_stack.items():
+                for group, gitr in lgitr.items():
+                    for loc, ritr in gitr.items():
+                        for region in ritr:
+                            data["system_tree_node"] += [tree]
+                            data["location_group"] += [group]
+                            data["location"] += [loc]
+                            data["region"] += [region.region]
+                            data["attributes"] += [region.attributes]
+                            data["depth"] += [region.depth]
+                            data["category"] += [region.category]
+                            data["name"] += [region.name]
+                            data["start_ts"] += [region.enter_nsec]
+                            data["end_ts"] += [region.leave_nsec]
+
+            return (trace, pd.DataFrame.from_dict(data))
+
+        readers = []
+        df = pd.DataFrame()
+        for itr in self.filename:
+            _reader, _df = _read_trace(itr)
+            readers += [_reader]
+            df = pd.concat([df, _df])
+
+        return (df, readers)
+
+
+def read_trace(filename):
+    data = OTF2Reader(filename).read()[0]
+
+    print(f"\nDATA:\n{data}")
+
+    attributes = list(data["attributes"])
+
+    print(f"\nATTRIBUTES:\n{attributes}")
@@ -48,3 +48,30 @@ def test_perfetto_data(
        assert len(_pf_data) == len(
            _js_data
        ), f"{pf_category} ({len(_pf_data)}):\n\t{_pf_data}\n{js_category} ({len(_js_data)}):\n\t{_js_data}"
+
+
+def test_otf2_data(
+    otf2_data, json_data, categories=("hip", "hsa", "marker", "kernel", "memory_copy")
+):
+
+    mapping = {
+        "hip": ("hip_api", "hip_api"),
+        "hsa": ("hsa_api", "hsa_api"),
+        "marker": ("marker_api", "marker_api"),
+        "kernel": ("kernel_dispatch", "kernel_dispatch"),
+        "memory_copy": ("memory_copy", "memory_copy"),
+    }
+
+    # make sure they specified valid categories
+    for itr in categories:
+        assert itr in mapping.keys()
+
+    for otf2_category, json_category in [
+        itr for key, itr in mapping.items() if key in categories
+    ]:
+        _otf2_data = otf2_data.loc[otf2_data["category"] == otf2_category]
+        _json_data = json_data["rocprofiler-sdk-tool"]["buffer_records"][json_category]
+
+        assert len(_otf2_data) == len(
+            _json_data
+        ), f"{otf2_category} ({len(_otf2_data)}):\n\t{_otf2_data}\n{json_category} ({len(_json_data)}):\n\t{_json_data}"
@@ -36,7 +36,7 @@ add_test(
        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> -M --sys-trace no --hsa-trace=0
        --hsa-core-trace=1 --hip-compiler-trace False --hip-runtime-trace --kernel-trace
        --memory-copy-trace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace-cmd-line -o out
-        --output-format pftrace json --log-level env --kernel-rename
+        --output-format pftrace json otf2 --log-level env --kernel-rename
        --perfetto-shmem-size-hint 128 --perfetto-buffer-size 2048000
        --perfetto-buffer-fill-policy ring_buffer --perfetto-backend inprocess --
        $<TARGET_FILE:transpose>)
@@ -52,7 +52,9 @@ add_test(
        ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.json
        --pftrace-input
-        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.pftrace)
+        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.pftrace
+        --otf2-input
+        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.otf2)

 set(VALIDATION_FILES
    ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-cmd-line/out_results.pftrace
@@ -94,7 +96,9 @@ add_test(
        ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --json-input
        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.json
        --pftrace-input
-        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.pftrace)
+        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.pftrace
+        --otf2-input
+        ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.otf2)

 set(VALIDATION_FILES
    ${CMAKE_CURRENT_BINARY_DIR}/transpose-trace-inp-yaml/out_results.pftrace
@@ -2,10 +2,12 @@

 import pytest
 import json
+import os

 from rocprofiler_sdk.pytest_utils.dotdict import dotdict
 from rocprofiler_sdk.pytest_utils import collapse_dict_list
 from rocprofiler_sdk.pytest_utils.perfetto_reader import PerfettoReader
+from rocprofiler_sdk.pytest_utils.otf2_reader import OTF2Reader


 def pytest_addoption(parser):
@@ -19,6 +21,11 @@ def pytest_addoption(parser):
        action="store",
        help="Path to Perfetto trace file.",
    )
+    parser.addoption(
+        "--otf2-input",
+        action="store",
+        help="Path to OTF2 trace file.",
+    )


@pytest.fixture
@@ -32,3 +39,11 @@ def json_data(request):
 def pftrace_data(request):
    filename = request.config.getoption("--pftrace-input")
    return PerfettoReader(filename).read()[0]
+
+
+@pytest.fixture
+def otf2_data(request):
+    filename = request.config.getoption("--otf2-input")
+    if not os.path.exists(filename):
+        raise FileExistsError(f"{filename} does not exist")
+    return OTF2Reader(filename).read()[0]
@@ -11,7 +11,7 @@ jobs:
    marker_trace: False
    output_directory: "@CMAKE_CURRENT_BINARY_DIR@/%argt%-trace-inp-yaml"
    output_file: out
-    output_format: [pftrace, json]
+    output_format: [pftrace, json, otf2]
    log_level: env
    kernel_rename: True
    perfetto_shmem_size_hint: 128
@@ -143,6 +143,14 @@ def test_perfetto_data(pftrace_data, json_data):
    )


+def test_otf2_data(otf2_data, json_data):
+    import rocprofiler_sdk.tests.rocprofv3 as rocprofv3
+
+    rocprofv3.test_otf2_data(
+        otf2_data, json_data, ("hip", "hsa", "kernel", "memory_copy")
+    )
+
+
 if __name__ == "__main__":
    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
    sys.exit(exit_code)
@@ -10,75 +10,31 @@ project(

 find_package(rocprofiler-sdk REQUIRED)

-add_test(
-    NAME rocprofv3-test-trace-hip-in-libraries-execute
-    COMMAND
-        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hip-runtime-trace --hsa-core-trace
-        --hsa-amd-trace --marker-trace --kernel-trace --memory-copy-trace --stats
-        --output-format csv -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out --
-        $<TARGET_FILE:hip-in-libraries>)
-
-add_test(
-    NAME rocprofv3-test-trace-hip-in-libraries-json-execute
-    COMMAND
-        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hip-runtime-trace --hsa-core-trace
-        --hsa-amd-trace --marker-trace --kernel-trace --memory-copy-trace --stats
-        --output-format JSON -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out --
-        $<TARGET_FILE:hip-in-libraries>)
-
-add_test(
-    NAME rocprofv3-test-trace-hip-in-libraries-pftrace-execute
-    COMMAND
-        $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hip-runtime-trace --hsa-core-trace
-        --hsa-amd-trace --marker-trace --kernel-trace --memory-copy-trace --stats
-        --output-format pftrace -d ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out --
-        $<TARGET_FILE:hip-in-libraries>)
+rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.py)

 string(REPLACE "LD_PRELOAD=" "ROCPROF_PRELOAD=" PRELOAD_ENV
               "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}")

 set(tracing-env "${PRELOAD_ENV}")

-set_tests_properties(
-    rocprofv3-test-trace-hip-in-libraries-execute
-    PROPERTIES
-        TIMEOUT
-        100
-        LABELS
-        "integration-tests"
-        ENVIRONMENT
-        "${tracing-env}"
-        FAIL_REGULAR_EXPRESSION
-        "HSA_CORE_API|HSA_AMD_EXT_API|HSA_IMAGE_EXT_API|HSA_FINALIZER_EXT_API|HIP_API|HIP_COMPILER_API|KERNEL_DISPATCH|CODE_OBJECT"
-    )
+set(VALIDATION_DEPENDS)
+foreach(_OUTPUT_FORMAT csv json pftrace otf2)
+    add_test(
+        NAME rocprofv3-test-trace-hip-in-libraries-${_OUTPUT_FORMAT}-execute
+        COMMAND
+            $<TARGET_FILE:rocprofiler-sdk::rocprofv3> --hip-runtime-trace
+            --hsa-core-trace --hsa-amd-trace --marker-trace --kernel-trace
+            --memory-copy-trace --stats --output-format ${_OUTPUT_FORMAT} -d
+            ${CMAKE_CURRENT_BINARY_DIR}/%argt%-trace -o out --
+            $<TARGET_FILE:hip-in-libraries>)

-set_tests_properties(
-    rocprofv3-test-trace-hip-in-libraries-json-execute
-    PROPERTIES
-        TIMEOUT
-        100
-        LABELS
-        "integration-tests"
-        ENVIRONMENT
-        "${tracing-env}"
-        FAIL_REGULAR_EXPRESSION
-        "HSA_CORE_API|HSA_AMD_EXT_API|HSA_IMAGE_EXT_API|HSA_FINALIZER_EXT_API|HIP_API|HIP_COMPILER_API|KERNEL_DISPATCH|CODE_OBJECT"
-    )
+    set_tests_properties(
+        rocprofv3-test-trace-hip-in-libraries-${_OUTPUT_FORMAT}-execute
+        PROPERTIES TIMEOUT 100 LABELS "integration-tests" ENVIRONMENT "${tracing-env}")

-set_tests_properties(
-    rocprofv3-test-trace-hip-in-libraries-pftrace-execute
-    PROPERTIES
-        TIMEOUT
-        100
-        LABELS
-        "integration-tests"
-        ENVIRONMENT
-        "${tracing-env}"
-        FAIL_REGULAR_EXPRESSION
-        "HSA_CORE_API|HSA_AMD_EXT_API|HSA_IMAGE_EXT_API|HSA_FINALIZER_EXT_API|HIP_API|HIP_COMPILER_API|KERNEL_DISPATCH|CODE_OBJECT"
-    )
-
-rocprofiler_configure_pytest_files(CONFIG pytest.ini COPY validate.py conftest.py)
+    list(APPEND VALIDATION_DEPENDS
+         rocprofv3-test-trace-hip-in-libraries-${_OUTPUT_FORMAT}-execute)
+endforeach()

 add_test(
    NAME rocprofv3-test-trace-hip-in-libraries-validate
@@ -103,7 +59,8 @@ add_test(
        ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_memory_copy_stats.csv
        --json-input ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.json
        --pftrace-input
-        ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.pftrace)
+        ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.pftrace
+        --otf2-input ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.otf2)

 set(VALIDATION_FILES
    ${CMAKE_CURRENT_BINARY_DIR}/hip-in-libraries-trace/out_results.pftrace
@@ -120,14 +77,13 @@ set(VALIDATION_FILES

 set_tests_properties(
    rocprofv3-test-trace-hip-in-libraries-validate
-    PROPERTIES
-        TIMEOUT
-        45
-        LABELS
-        "integration-tests"
-        DEPENDS
-        "rocprofv3-test-trace-hip-in-libraries-execute;rocprofv3-test-trace-hip-in-libraries-json-execute;rocprofv3-test-trace-hip-in-libraries-pftrace-execute"
-        FAIL_REGULAR_EXPRESSION
-        "AssertionError"
-        ATTACHED_FILES_ON_FAIL
-        "${VALIDATION_FILES}")
+    PROPERTIES TIMEOUT
+               45
+               LABELS
+               "integration-tests"
+               DEPENDS
+               "${VALIDATION_DEPENDS}"
+               FAIL_REGULAR_EXPRESSION
+               "AssertionError"
+               ATTACHED_FILES_ON_FAIL
+               "${VALIDATION_FILES}")
@@ -8,6 +8,7 @@ import json
 from rocprofiler_sdk.pytest_utils.dotdict import dotdict
 from rocprofiler_sdk.pytest_utils import collapse_dict_list
 from rocprofiler_sdk.pytest_utils.perfetto_reader import PerfettoReader
+from rocprofiler_sdk.pytest_utils.otf2_reader import OTF2Reader


 def pytest_addoption(parser):
@@ -61,7 +62,6 @@ def pytest_addoption(parser):
        action="store",
        help="Path to memory copy stats CSV file.",
    )
-
    parser.addoption(
        "--json-input",
        action="store",
@@ -72,6 +72,11 @@ def pytest_addoption(parser):
        action="store",
        help="Path to Perfetto trace file.",
    )
+    parser.addoption(
+        "--otf2-input",
+        action="store",
+        help="Path to OTF2 trace file.",
+    )


@pytest.fixture
@@ -210,3 +215,9 @@ def json_data(request):
 def pftrace_data(request):
    filename = request.config.getoption("--pftrace-input")
    return PerfettoReader(filename).read()[0]
+
+
+@pytest.fixture
+def otf2_data(request):
+    filename = request.config.getoption("--otf2-input")
+    return OTF2Reader(filename).read()[0]
@@ -438,6 +438,13 @@ def test_perfetto_data(pftrace_data, json_data):
    )


+def test_otf2_data(otf2_data, json_data):
+    import rocprofiler_sdk.tests.rocprofv3 as rocprofv3
+
+    # do not test for HSA and HIP since that may vary slightly b/t two separate runs
+    rocprofv3.test_otf2_data(otf2_data, json_data, ("marker", "kernel", "memory_copy"))
+
+
 if __name__ == "__main__":
    exit_code = pytest.main(["-x", __file__] + sys.argv[1:])
    sys.exit(exit_code)