Kernel Tracing Fix (#439)

* Update lib/rocprofiler-sdk/hsa/queue.cpp

- switch using the kernel_pkt.kernel_dispatch.completion_signal instead of interrupt signal for getting the dispatch time

* Update tests/kernel-tracing/validate.py

- add verification of total runtime collected in test_timestamps
  - the sum of the runtime of all the kernels in reproducible-runtime should be ~1 sec +/- 10%

* Remove include/rocprofiler-sdk/rocprofiler_plugin.h

* Update CI workflow

- update actions/cache@v3 -> v4
- actions/cache/save@v3 -> v4
- thollander/actions-comment-pull-request@v2 -> v2.4.3

* Update pytest.ini

- change default options to one that is more verbose

* Update tests/kernel-tracing/CMakeLists.txt

- skip test_total_runtime when Address or Thread Sanitizer enabled
  - overhead skews the results

* Update tests/kernel-tracing/validate.py

- separate test_total_runtime test

tests/async-copy-tracing/pytest.ini

@@ -1,4 +1,4 @@
 
 [pytest]
-addopts = --durations=20 -ras -vv
+addopts = --durations=20 -rA -s -vv
 testpaths = validate.py

tests/kernel-tracing/CMakeLists.txt

@@ -10,6 +10,11 @@ project(
 
 find_package(rocprofiler-sdk REQUIRED)
 
+set(PYTEST_ARGS)
+if(ROCPROFILER_MEMCHECK MATCHES "(Address|Thread)Sanitizer")
+    set(PYTEST_ARGS -k "not test_total_runtime")
+endif()
+
 if(ROCPROFILER_MEMCHECK_PRELOAD_ENV)
     set(PRELOAD_ENV
         "${ROCPROFILER_MEMCHECK_PRELOAD_ENV}:$<TARGET_FILE:rocprofiler-sdk-json-tool>")
@@ -36,9 +41,10 @@ foreach(FILENAME validate.py pytest.ini conftest.py)
                    ${CMAKE_CURRENT_BINARY_DIR}/${FILENAME} COPYONLY)
 endforeach()
 
-add_test(NAME test-kernel-tracing-validate
-         COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py --input
-                 ${CMAKE_CURRENT_BINARY_DIR}/kernel-tracing-test.json)
+add_test(
+    NAME test-kernel-tracing-validate
+    COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/validate.py ${PYTEST_ARGS}
+            --input ${CMAKE_CURRENT_BINARY_DIR}/kernel-tracing-test.json)
 
 set_tests_properties(
     test-kernel-tracing-validate

tests/kernel-tracing/pytest.ini

@@ -1,4 +1,4 @@
 
 [pytest]
-addopts = --durations=20 -ras -vv
+addopts = --durations=20 -rA -s -vv
 testpaths = validate.py

tests/kernel-tracing/validate.py

@@ -74,6 +74,20 @@ def test_timestamps(input_data):
         assert api_end <= itr["end_timestamp"]
 
 
+def test_total_runtime(input_data):
+    sdk_data = input_data["rocprofiler-sdk-json-tool"]
+
+    runtime_data = []
+    for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
+        elapsed = itr["end_timestamp"] - itr["start_timestamp"]
+        runtime_data.append(elapsed)  # in nanoseconds
+
+    expected_runtime = 1.0e3  # one second in milliseconds
+
+    assert (sum(runtime_data) * 1.0e-6) >= (0.9 * expected_runtime)
+    assert (sum(runtime_data) * 1.0e-6) <= (1.1 * expected_runtime)
+
+
 def test_internal_correlation_ids(input_data):
     data = input_data
     sdk_data = data["rocprofiler-sdk-json-tool"]

tests/rocprofv3/counter-collection/pytest.ini

@@ -1,5 +1,5 @@
 
 [pytest]
-addopts = --durations=20 -ras -vv
+addopts = --durations=20 -rA -s -vv
 testpaths = input1/validate.py
             input2/validate.py

tests/rocprofv3/tracing-plus-cc/pytest.ini

@@ -1,4 +1,4 @@
 
 [pytest]
-addopts = --durations=20 -ras -vv
+addopts = --durations=20 -rA -s -vv
 testpaths = validate.py