Update Perfetto and fix tests (#378)

Fix for "SWDEV-479652" - Perfetto-based tests are failing.

Updated version of perfetto submodule to v46.0.
Modified Omnitrace code that uses Perfetto, so it can compile.
Modified the testing code, so it can run the version of trace_processor_shell provided (v46.0).

---------

Signed-off-by: Aleksandar Janicijevic <Aleksandar.Janicijevic@amd.com>
Co-authored-by: David Galiffi <David.Galiffi@amd.com>

.github/workflows/opensuse.yml

@@ -58,6 +58,11 @@ jobs:
         timeout_minutes: 25
         max_attempts: 5
         command: |
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
+          python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
 
     - name: Configure Env
@@ -87,6 +92,7 @@ jobs:
           -DOMNITRACE_USE_HIP=OFF
           -DOMNITRACE_USE_OMPT=OFF
           -DOMNITRACE_USE_PYTHON=ON
+          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=OFF
           -DOMNITRACE_USE_MPI_HEADERS=ON
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"

.github/workflows/redhat.yml

@@ -68,6 +68,16 @@ jobs:
         env
 
     - name: Install Packages
+      shell: bash
+      run: |
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
+          python3 -m pip install 'cmake==3.21.4' &&
+          for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
+
+    - name: Install ROCm Packages
       if: ${{ matrix.rocm-version > 0 }}
       timeout-minutes: 30
       shell: bash
@@ -109,6 +119,7 @@ jobs:
           -DOMNITRACE_USE_MPI_HEADERS=ON
           -DOMNITRACE_CI_MPI_RUN_AS_ROOT=ON
           -DOMNITRACE_MAX_THREADS=64
+          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=OFF
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"
           -DOMNITRACE_DISABLE_EXAMPLES="transpose;rccl"

.github/workflows/ubuntu-focal.yml

@@ -94,9 +94,10 @@ jobs:
           apt-get update &&
           apt-get upgrade -y &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libmpich-dev mpich environment-modules ${{ matrix.compiler }} &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.16.3' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
@@ -246,20 +247,12 @@ jobs:
         mpi-headers: ['OFF']
         build-jobs: ['3']
         ctest-exclude: ['-LE "mpi-example|transpose"']
-        perfetto-tools: ['OFF']
         include:
-          - compiler: 'g++'
-            rocm-version: '4.5'
-            mpi-headers: 'OFF'
-            build-jobs: '2'
-            ctest-exclude: '-LE "mpi-example|transpose"'
-            perfetto-tools: 'ON'
           - compiler: 'g++'
             rocm-version: 'debian'
             mpi-headers: 'ON'
             build-jobs: '2'
             ctest-exclude: '-LE transpose'
-            perfetto-tools: 'OFF'
 
     env:
       BUILD_TYPE: MinSizeRel
@@ -283,7 +276,10 @@ jobs:
           echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm-version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list &&
           apt-get update &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev1 libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev rccl-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
@@ -345,7 +341,6 @@ jobs:
           -DOMNITRACE_USE_PYTHON=ON
           -DOMNITRACE_USE_MPI_HEADERS=${{ matrix.mpi-headers }}
           -DOMNITRACE_USE_SANITIZER=OFF
-          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=${{ matrix.perfetto-tools }}
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"
           -DOMNITRACE_CI_MPI_RUN_AS_ROOT=${{ matrix.mpi-headers }}
@@ -444,9 +439,10 @@ jobs:
           sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules ${{ matrix.deps }} ${{ matrix.compiler }} &&
           if [ "${{ matrix.mpi }}" = "mpich" ]; then sudo apt-get install -y libmpich-dev mpich; fi &&
           if [ "${{ matrix.mpi }}" = "openmpi" ]; then sudo apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.16.3' &&
           sudo apt-get -y --purge autoremove &&
           sudo apt-get -y clean
@@ -596,9 +592,10 @@ jobs:
         command: |
           apt-get update &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules gcc g++ mpich libmpich-dev texinfo &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.24.1' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&

.github/workflows/ubuntu-jammy.yml

@@ -110,8 +110,10 @@ jobs:
           apt-get install -y software-properties-common &&
           apt-get upgrade -y &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libopenmpi-dev libfabric-dev openmpi-bin environment-modules ${{ matrix.compiler }} &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
 

cmake/Packages.cmake

@@ -483,7 +483,7 @@ omnitrace_checkout_git_submodule(
     RELATIVE_PATH external/perfetto
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     REPO_URL https://android.googlesource.com/platform/external/perfetto
-    REPO_BRANCH v28.0
+    REPO_BRANCH v46.0
     TEST_FILE sdk/perfetto.cc)
 
 include(Perfetto)

cmake/Perfetto.cmake

@@ -67,10 +67,10 @@ if(CMAKE_CXX_COMPILER_IS_CLANG)
 else()
     set(PERFETTO_IS_CLANG false)
     set(OMNITRACE_PERFETTO_C_FLAGS
-        "-static-libgcc -Wno-maybe-uninitialized"
+        "-static-libgcc -Wno-maybe-uninitialized -Wno-stringop-overflow"
         CACHE STRING "Perfetto C flags")
     set(OMNITRACE_PERFETTO_CXX_FLAGS
-        "-static-libgcc -Wno-maybe-uninitialized"
+        "-static-libgcc -Wno-maybe-uninitialized -Wno-stringop-overflow -Wno-mismatched-new-delete"
         CACHE STRING "Perfetto C++ flags")
 endif()
 

cmake/Templates/args.gn.in

@@ -8,19 +8,19 @@ is_clang = @PERFETTO_IS_CLANG@
 is_hermetic_clang = false
 
 enable_perfetto_benchmarks = false
-enable_perfetto_integration_tests = false
+enable_perfetto_integration_tests = true
 enable_perfetto_unittests = false
 enable_perfetto_fuzzers = false
 
 # enable_perfetto_stderr_crash_dump = false
 enable_perfetto_heapprofd = false
 enable_perfetto_tools = false
-enable_perfetto_trace_processor = false
-enable_perfetto_trace_processor_httpd = false
+enable_perfetto_trace_processor = true
+enable_perfetto_trace_processor_httpd = true
 enable_perfetto_trace_processor_json = false
 enable_perfetto_trace_processor_linenoise = false
 enable_perfetto_trace_processor_percentile = false
-enable_perfetto_trace_processor_sqlite = false
+enable_perfetto_trace_processor_sqlite = true
 enable_perfetto_ui = false
 
 extra_cflags = "@OMNITRACE_PERFETTO_C_FLAGS@"

examples/python/source-numpy.py

@@ -0,0 +1,83 @@
+#!@PYTHON_EXECUTABLE@
+
+import os
+import sys
+import time
+import omnitrace
+from omnitrace.user import region as omni_user_region
+
+_prefix = ""
+
+
+def fib(n):
+    return n if n < 2 else (fib(n - 1) + fib(n - 2))
+
+
+try:
+    import numpy as np
+
+    def inefficient(n):
+        print(f"[{_prefix}] ... running inefficient({n}) (1)")
+        a = 0
+        for i in range(n):
+            a += i
+            for j in range(n):
+                a += j
+        _len = a * n * n
+        _ret = np.random.rand(_len).sum()
+        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+        return _ret
+
+except ImportError as e:
+    print(f"ImportError: {e}")
+    import random
+
+    def _sum(arr):
+        print(f"----  in _sum")
+        return sum(arr)
+
+    def inefficient(n):
+        print(f"[{_prefix}] ... running inefficient({n})")
+        a = 0
+        for i in range(n):
+            a += i
+            for j in range(n):
+                a += j
+        _len = a * n * n
+        _arr = [random.random() for _ in range(_len)]
+        _ret = _sum(_arr)
+        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+        return _ret
+
+
+@omnitrace.profile()
+def run(n):
+    _ret = 0
+    _ret += fib(n)
+    _ret += inefficient(n)
+    return _ret
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-n", "--num-iterations", help="Number", type=int, default=3)
+    parser.add_argument("-v", "--value", help="Starting value", type=int, default=20)
+    parser.add_argument(
+        "-s",
+        "--stop-profile",
+        help="Stop tracing after given iterations",
+        type=int,
+        default=0,
+    )
+    args = parser.parse_args()
+
+    _prefix = os.path.basename(__file__)
+    print(f"[{_prefix}] Executing {args.num_iterations} iterations...\n")
+    for i in range(args.num_iterations):
+        with omni_user_region(f"main_loop"):
+            if args.stop_profile > 0 and i == args.stop_profile:
+                omnitrace.user.stop_trace()
+            ans = run(args.value)
+            print(f"[{_prefix}] [{i}] result of run({args.value}) = {ans}\n")

examples/python/source.py

@@ -5,6 +5,7 @@ import sys
 import time
 import omnitrace
 from omnitrace.user import region as omni_user_region
+import random
 
 _prefix = ""
 
@@ -13,39 +14,23 @@ def fib(n):
     return n if n < 2 else (fib(n - 1) + fib(n - 2))
 
 
-try:
-    import numpy as np
+def _sum(arr):
+    print(f"----  in _sum")
+    return sum(arr)
 
-    def inefficient(n):
-        print(f"[{_prefix}] ... running inefficient({n})")
-        a = 0
-        for i in range(n):
-            a += i
-            for j in range(n):
-                a += j
-        _len = a * n * n
-        _ret = np.random.rand(_len).sum()
-        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
-        return _ret
 
-except ImportError:
-    import random
-
-    def _sum(arr):
-        return sum(arr)
-
-    def inefficient(n):
-        print(f"[{_prefix}] ... running inefficient({n})")
-        a = 0
-        for i in range(n):
-            a += i
-            for j in range(n):
-                a += j
-        _len = a * n * n
-        _arr = [random.random() for _ in range(_len)]
-        _ret = _sum(_arr)
-        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
-        return _ret
+def inefficient(n):
+    print(f"[{_prefix}] ... running inefficient({n})")
+    a = 0
+    for i in range(n):
+        a += i
+        for j in range(n):
+            a += j
+    _len = a * n * n
+    _arr = [random.random() for _ in range(_len)]
+    _ret = _sum(_arr)
+    print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+    return _ret
 
 
 @omnitrace.profile()

source/lib/core/perfetto.hpp

@@ -109,11 +109,12 @@ perfetto_counter_track<Tp>::emplace(size_t _idx, const std::string& _v,
     auto        _index     = _track_data.size();
     auto&       _name      = _name_data.emplace_back(std::make_unique<std::string>(_v));
     const char* _unit_name = (_units && strlen(_units) > 0) ? _units : nullptr;
-    _track_data.emplace_back(::perfetto::CounterTrack{ _name->c_str() }
-                                 .set_unit_name(_unit_name)
-                                 .set_category(_category)
-                                 .set_unit_multiplier(_mult)
-                                 .set_is_incremental(_incr));
+    _track_data.emplace_back(
+        ::perfetto::CounterTrack{ ::perfetto::DynamicString{ _name->c_str() } }
+            .set_unit_name(_unit_name)
+            .set_category(_category)
+            .set_unit_multiplier(_mult)
+            .set_is_incremental(_incr));
     if(config::get_is_continuous_integration())
     {
         for(auto& itr : _missing)

source/lib/omnitrace/library/tracing.hpp

@@ -642,14 +642,14 @@ mark_perfetto_ts(CategoryT, const char* name, uint64_t _ts, Args&&... args)
 
 template <typename CategoryT, typename... Args>
 inline void
-mark_perfetto_track(CategoryT, const char*, ::perfetto::Track _track, uint64_t _ts,
+mark_perfetto_track(CategoryT, const char* name, ::perfetto::Track _track, uint64_t _ts,
                     Args&&... args)
 {
     // skip if category is disabled
     if(category_mark_disabled<CategoryT>()) return;
 
-    TRACE_EVENT_INSTANT(trait::name<CategoryT>::value, _track, _ts,
-                        std::forward<Args>(args)...);
+    TRACE_EVENT_INSTANT(trait::name<CategoryT>::value, ::perfetto::DynamicString{ name },
+                        _track, _ts, std::forward<Args>(args)...);
 }
 }  // namespace tracing
 }  // namespace omnitrace

tests/omnitrace-python-tests.cmake

@@ -166,7 +166,8 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
             NAME ${TEST_NAME}-validate-perfetto
             COMMAND
                 ${_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/validate-perfetto-proto.py
-                -m ${TEST_PERFETTO_METRIC} ${TEST_ARGS} -p -i
+                -m ${TEST_PERFETTO_METRIC} ${TEST_ARGS} -p -t
+                /opt/trace_processor/bin/trace_processor_shell -i
             PYTHON_VERSION ${_VERSION}
             FILE omnitrace-tests-output/${TEST_NAME}/${_VERSION}/${TEST_PERFETTO_FILE}
             DEPENDS ${TEST_NAME}-${_VERSION}

tests/omnitrace-testing.cmake

@@ -985,6 +985,7 @@ function(OMNITRACE_ADD_VALIDATION_TEST)
                 ${CMAKE_CURRENT_LIST_DIR}/validate-perfetto-proto.py -m
                 "${TEST_PERFETTO_METRIC}" ${TEST_ARGS} -i
                 ${PROJECT_BINARY_DIR}/omnitrace-tests-output/${TEST_NAME}/${TEST_PERFETTO_FILE}
+                -t /opt/trace_processor/bin/trace_processor_shell
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     endif()
 

tests/validate-perfetto-proto.py

@@ -1,23 +1,36 @@
 #!/usr/bin/env python3
 
 import sys
+import os
 import argparse
-from perfetto.trace_processor import TraceProcessor
+from perfetto.trace_processor import TraceProcessor, TraceProcessorConfig
 
 
-def load_trace(inp, max_tries=5, retry_wait=1):
+def load_trace(inp, max_tries=5, retry_wait=1, bin_path=None):
     """Occasionally connecting to the trace processor fails with HTTP errors
     so this function tries to reduce spurious test failures"""
 
     n = 0
     tp = None
+
+    # Check if bin_path is set and if it exists
+    print("trace_processor path: ", bin_path)
+    if bin_path and not os.path.isfile(bin_path):
+        print(f"Path {bin_path} does not exist. Using the default path.")
+        bin_path = None
+
     while tp is None:
         try:
-            tp = TraceProcessor(trace=(inp))
+            if bin_path:
+                config = TraceProcessorConfig(bin_path=bin_path)
+                tp = TraceProcessor(trace=inp, config=config)
+            else:
+                tp = TraceProcessor(trace=inp)
             break
-        except Exception as e:
-            sys.stderr.write(f"{e}\n")
+        except Exception as ex:
+            sys.stderr.write(f"{ex}\n")
             sys.stderr.flush()
+
             if n >= max_tries:
                 raise
             else:
@@ -71,6 +84,9 @@ if __name__ == "__main__":
         "-p", "--print", action="store_true", help="Print the processed perfetto data"
     )
     parser.add_argument("-i", "--input", type=str, help="Input file", required=True)
+    parser.add_argument(
+        "-t", "--trace_processor_shell", type=str, help="Path of trace_processor_shell"
+    )
     parser.add_argument(
         "--key-names",
         type=str,
@@ -93,7 +109,7 @@ if __name__ == "__main__":
             "The same number of labels, counts, and depths must be specified"
         )
 
-    tp = load_trace(args.input)
+    tp = load_trace(args.input, bin_path=args.trace_processor_shell)
 
     if tp is None:
         raise ValueError(f"trace {args.input} could not be loaded")