roctracer: device and stream tracks (#209)

* roctracer: use multiple tracks for HIP streams

Use different perfetto tracks for each stream, and set the name of
these tracks to the stream pointer values. Setting the name like this
matches the args in the API traces.
This fixes overlapping work on multiple streams appearing as a call
stack.

* Fix -pedantic

* Run clang-format

* Add option to disable per stream tracks in perfetto

* Updated scheme for roctracer activity + general roctracer fixes

- Per-device tracks
- Handle HSA OPS in ROCm 5.3
  - the changes in ROCm 5.3 were causing HSA ops to get discarded
- Default for OMNITRACE_ROCTRACER_DISCARD_INVALID is now zero
  - i.e. default behavior is to flip beg_ns and end_ns when beg_ns > end_ns

* Flush perfetto at end of hip_activity_callback

- fixes unterminated regions

* GitHub Actions and run-ci script updates

- improve reliability

* Set OMNITRACE_TMPDIR in testing

- files in /tmp get occasionally deleted during CI

Co-authored-by: Gergely Meszaros <gergely@streamhpc.com>

scripts/run-ci.py

@@ -4,6 +4,7 @@
 import os
 import re
 import sys
+import glob
 import socket
 import shutil
 import argparse
@@ -240,15 +241,34 @@ if __name__ == "__main__":
     for itr in args.stages:
         dashboard_args.append(f"{args.mode}{itr}")
 
-    run(
-        [CTEST_CMD]
-        + dashboard_args
-        + [
-            "-S",
-            os.path.join(args.binary_dir, "dashboard.cmake"),
-            "--output-on-failure",
-            "-VV",
-        ]
-        + ctest_args,
-        check=True,
-    )
+    try:
+        run(
+            [CTEST_CMD]
+            + dashboard_args
+            + [
+                "-S",
+                os.path.join(args.binary_dir, "dashboard.cmake"),
+                "--output-on-failure",
+                "-V",
+            ]
+            + ctest_args,
+            check=True,
+        )
+    finally:
+        if "-VV" not in ctest_args:
+            for file in glob.glob(
+                os.path.join(args.binary_dir, "Testing/**"), recursive=True
+            ):
+                if not os.path.isfile(file):
+                    continue
+                print(f"\n\n\n###### Reading {file}... ######\n\n\n")
+                with open(file, "r") as inpf:
+                    fdata = inpf.read()
+                    if "LastTest" not in file and "Coverage" not in file:
+                        print(fdata)
+                    oname = os.path.basename(file)
+                    if oname.endswith(".log"):
+                        oname += ".log"
+                    with open(os.path.join(args.binary_dir, oname), "w") as outf:
+                        print(f"\n\n###### Writing {oname}... ######\n\n")
+                        outf.write(fdata)