roctracer: device and stream tracks (#209)

* roctracer: use multiple tracks for HIP streams

Use different perfetto tracks for each stream, and set the name of
these tracks to the stream pointer values. Setting the name like this
matches the args in the API traces.
This fixes overlapping work on multiple streams appearing as a call
stack.

* Fix -pedantic

* Run clang-format

* Add option to disable per stream tracks in perfetto

* Updated scheme for roctracer activity + general roctracer fixes

- Per-device tracks
- Handle HSA OPS in ROCm 5.3
  - the changes in ROCm 5.3 were causing HSA ops to get discarded
- Default for OMNITRACE_ROCTRACER_DISCARD_INVALID is now zero
  - i.e. default behavior is to flip beg_ns and end_ns when beg_ns > end_ns

* Flush perfetto at end of hip_activity_callback

- fixes unterminated regions

* GitHub Actions and run-ci script updates

- improve reliability

* Set OMNITRACE_TMPDIR in testing

- files in /tmp get occasionally deleted during CI

Co-authored-by: Gergely Meszaros <gergely@streamhpc.com>

.github/workflows/cpack.yml

@@ -161,16 +161,11 @@ jobs:
           echo "CI_SCRIPT_ARGS=--rocm +python" >> $GITHUB_ENV
 
       - name: Build Base Container
-        timeout-minutes: 20
-        uses: nick-fields/retry@v2
-        with:
-          retry_wait_seconds: 120
-          timeout_minutes: 20
-          max_attempts: 5
-          command: |
-            pushd docker
-            ./build-docker.sh --distro ${{ matrix.os-distro }} --versions ${{ matrix.os-version }} --rocm-versions ${{ matrix.rocm-version }}
-            popd
+        timeout-minutes: 30
+        run: |
+          pushd docker
+          ./build-docker.sh --distro ${{ matrix.os-distro }} --versions ${{ matrix.os-version }} --rocm-versions ${{ matrix.rocm-version }}
+          popd
 
       - name: Build Release
         timeout-minutes: 150

.github/workflows/opensuse.yml

@@ -20,6 +20,7 @@ concurrency:
 env:
   OMNITRACE_VERBOSE: 1
   OMNITRACE_CI: ON
+  OMNITRACE_TMPDIR: "%env{PWD}%/testing-tmp"
 
 jobs:
   opensuse:
@@ -37,11 +38,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done

.github/workflows/ubuntu-bionic.yml

@@ -23,6 +23,7 @@ env:
   OMNITRACE_VERBOSE: 1
   OMNITRACE_CI: ON
   GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
+  OMNITRACE_TMPDIR: "%env{PWD}%/testing-tmp"
 
 jobs:
   ubuntu-bionic:
@@ -37,7 +38,7 @@ jobs:
 
     steps:
     - name: Patch Git
-      timeout-minutes: 10
+      timeout-minutes: 25
       run: |
         apt-get update
         apt-get install -y software-properties-common
@@ -51,11 +52,11 @@ jobs:
         submodules: recursive
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           apt-get update &&

.github/workflows/ubuntu-focal.yml

@@ -20,6 +20,7 @@ concurrency:
 env:
   OMNITRACE_VERBOSE: 1
   OMNITRACE_CI: ON
+  OMNITRACE_TMPDIR: "%env{PWD}%/testing-tmp"
 
 jobs:
   ubuntu-focal-external:
@@ -67,11 +68,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           apt-get update &&
@@ -244,11 +245,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           apt-get update &&
@@ -408,11 +409,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           sudo apt-get update &&
@@ -555,11 +556,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           apt-get update &&

.github/workflows/ubuntu-jammy.yml

@@ -20,6 +20,7 @@ concurrency:
 env:
   OMNITRACE_VERBOSE: 1
   OMNITRACE_CI: ON
+  OMNITRACE_TMPDIR: "%env{PWD}%/testing-tmp"
 
 jobs:
   ubuntu-jammy-external:
@@ -70,11 +71,11 @@ jobs:
     - uses: actions/checkout@v3
 
     - name: Install Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         command: |
           apt-get update &&
@@ -87,12 +88,12 @@ jobs:
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
 
     - name: Install ROCm Packages
-      timeout-minutes: 10
+      timeout-minutes: 25
       if: ${{ matrix.rocm-version != '0.0' }}
       uses: nick-fields/retry@v2
       with:
         retry_wait_seconds: 30
-        timeout_minutes: 10
+        timeout_minutes: 25
         max_attempts: 5
         shell: bash
         command: |