Callback tracing for kernel dispatches + External correlation ID request service (#682)

* Support ROCPROFILER_CALLBACK_TRACING_KERNEL_DISPATCH

* Fix doxygen

* Update callback tracing

- temporary hacks for kind operation name and iterate kind operations

* Update source/include/rocprofiler-sdk

- introduce sequence id for kernel dispatches

* Update lib/rocprofiler-sdk (seq id)

- support sequence id passing

* Update tests (seq id)

- testing for sequence ids

* Cleanup include/rocprofiler-sdk/fwd.h

* Misc cleanup

* External Correlation ID Request Service (#699)

* External correlation ID request service

- callback requesting an external correlation ID instead of fetching from top of pushed external correlation ID stack

* Update external correlation id request support

- pass internal correlation ID in callback
- async copy generates a correlation ID if none already exists
- added external correlation ID request support for scratch memory tracing
- updated scratch memory tracing to use tracing:: functions

* Update hsa/queue.hpp

- new line at EOF

* Misc tweaks

- remove unnecessary logging in agent.cpp
- correlation_id::add_ref_count check for retirement
- finalization check in HSA queue AsyncSignalHandler

* Improve assertion failure logging in misc tests

* Update include/rocprofiler-sdk/fwd.h

- remove rocprofiler_record_counter_header_t

* Move lib/rocprofiler-sdk/tracing.hpp into lib/rocprofiler-sdk/tracing/ folder

* Update lib/rocprofiler-sdk/hsa/*

- hsa::get_hsa_status_string
- queue_info_session.hpp header
- rocprofiler_packet.hpp

* Update lib/rocprofiler-sdk/{counters,hip,marker}

- execute_phase_exit_callbacks tweaks
- queue_info_session tweaks

* Move rocprofiler_kernel_dispatch_operation_t to include/rocprofiler-sdk/fwd.h

* Update rocprofiler_buffer_tracing_kernel_dispatch_record_t

- add operation field and thread_id field

* Add lib/rocprofiler-sdk/kernel_dispatch

- enum <-> string mapping for kernel dispatch
- tracing implementations

* Update lib/rocprofiler-sdk/CMakeLists.txt

- tracing and kernel dispatch sub-directories

* Update lib/rocprofiler-sdk/{buffer,callback}_tracing.cpp

- invoke rocprofiler::kernel_tracing functions

* Update tests/common/serialization.hpp

- support operation and thread_id fields for rocprofiler_buffer_tracing_kernel_dispatch_record_t

* Update tests/tools/json-tool.cpp

- use external correlation id request service

* Rename sequence_id to dispatch_id

tests/kernel-tracing/validate.py

@@ -37,6 +37,7 @@ def test_data_structure(input_data):
     node_exists("hsa_api_traces", sdk_data["callback_records"])
     node_exists("hip_api_traces", sdk_data["callback_records"], 0)
     node_exists("marker_api_traces", sdk_data["callback_records"])
+    node_exists("kernel_dispatches", sdk_data["callback_records"])
 
     node_exists("names", sdk_data["buffer_records"])
     node_exists("kernel_dispatches", sdk_data["buffer_records"])
@@ -47,6 +48,32 @@ def test_data_structure(input_data):
     node_exists("retired_correlation_ids", sdk_data["buffer_records"])
 
 
+def test_size_entries(input_data):
+    # check that size fields are > 0 but account for function arguments
+    # which are named "size"
+    def check_size(data, bt):
+        if "size" in data.keys():
+            if isinstance(data["size"], str) and bt.endswith('["args"]'):
+                pass
+            else:
+                assert data["size"] > 0, f"origin: {bt}"
+
+    # recursively check the entire data structure
+    def iterate_data(data, bt):
+        if isinstance(data, (list, tuple)):
+            for i, itr in enumerate(data):
+                if isinstance(itr, dict):
+                    check_size(itr, f"{bt}[{i}]")
+                iterate_data(itr, f"{bt}[{i}]")
+        elif isinstance(data, dict):
+            check_size(data, f"{bt}")
+            for key, itr in data.items():
+                iterate_data(itr, f'{bt}["{key}"]')
+
+    # start recursive check over entire JSON dict
+    iterate_data(input_data, "input_data")
+
+
 def test_timestamps(input_data):
     data = input_data
     sdk_data = data["rocprofiler-sdk-json-tool"]
@@ -70,18 +97,26 @@ def test_timestamps(input_data):
 
     for titr in ["kernel_dispatches", "memory_copies"]:
         for itr in sdk_data["buffer_records"][titr]:
-            assert itr["start_timestamp"] < itr["end_timestamp"]
-            assert itr["correlation_id"]["internal"] > 0
-            assert itr["correlation_id"]["external"] > 0
-            assert sdk_data["metadata"]["init_time"] < itr["start_timestamp"]
-            assert sdk_data["metadata"]["init_time"] < itr["end_timestamp"]
-            assert sdk_data["metadata"]["fini_time"] > itr["start_timestamp"]
-            assert sdk_data["metadata"]["fini_time"] > itr["end_timestamp"]
+            assert itr["start_timestamp"] < itr["end_timestamp"], f"[{titr}] {itr}"
+            assert itr["correlation_id"]["internal"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["init_time"] < itr["start_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["init_time"] < itr["end_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["fini_time"] > itr["start_timestamp"]
+            ), f"[{titr}] {itr}"
+            assert (
+                sdk_data["metadata"]["fini_time"] > itr["end_timestamp"]
+            ), f"[{titr}] {itr}"
 
             api_start = cb_start[itr["correlation_id"]["internal"]]
             api_end = cb_end[itr["correlation_id"]["internal"]]
-            assert api_start < itr["start_timestamp"]
-            assert api_end <= itr["end_timestamp"]
+            assert api_start < itr["start_timestamp"], f"[{titr}] {itr}"
+            assert api_end <= itr["end_timestamp"], f"[{titr}] {itr}"
 
 
 def test_total_runtime(input_data):
@@ -157,13 +192,15 @@ def test_retired_correlation_ids(input_data):
 
     for cid, itr in async_corr_ids.items():
         assert cid in retired_corr_ids.keys()
-        ts = retired_corr_ids[cid]["timestamp"]
-        assert (ts - itr["end_timestamp"]) > 0, f"correlation-id: {cid}, data: {itr}"
+        retired_ts = retired_corr_ids[cid]["timestamp"]
+        end_ts = itr["end_timestamp"]
+        assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
 
     for cid, itr in api_corr_ids.items():
         assert cid in retired_corr_ids.keys()
-        ts = retired_corr_ids[cid]["timestamp"]
-        assert (ts - itr["end_timestamp"]) > 0, f"correlation-id: {cid}, data: {itr}"
+        retired_ts = retired_corr_ids[cid]["timestamp"]
+        end_ts = itr["end_timestamp"]
+        assert (retired_ts - end_ts) > 0, f"correlation-id: {cid}, data: {itr}"
 
     assert len(api_corr_ids.keys()) == (len(retired_corr_ids.keys()))
 
@@ -184,18 +221,17 @@ def test_external_correlation_ids(input_data):
     extern_corr_ids = list(set(sorted(extern_corr_ids)))
     for titr in ["hsa_api_traces", "marker_api_traces", "hip_api_traces"]:
         for itr in sdk_data["buffer_records"][titr]:
-            assert itr["correlation_id"]["external"] > 0
-            assert itr["thread_id"] == itr["correlation_id"]["external"]
-            assert itr["thread_id"] in extern_corr_ids
-            assert itr["correlation_id"]["external"] in extern_corr_ids
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert (
+                itr["thread_id"] == itr["correlation_id"]["external"]
+            ), f"[{titr}] {itr}"
+            assert itr["thread_id"] in extern_corr_ids, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
 
-    for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
-        assert itr["correlation_id"]["external"] > 0
-        assert itr["correlation_id"]["external"] in extern_corr_ids
-
-    for itr in sdk_data["buffer_records"]["memory_copies"]:
-        assert itr["correlation_id"]["external"] > 0
-        assert itr["correlation_id"]["external"] in extern_corr_ids
+    for titr in ["kernel_dispatches", "memory_copies"]:
+        for itr in sdk_data["buffer_records"][titr]:
+            assert itr["correlation_id"]["external"] > 0, f"[{titr}] {itr}"
+            assert itr["correlation_id"]["external"] in extern_corr_ids, f"[{titr}] {itr}"
 
 
 def test_kernel_ids(input_data):
@@ -220,6 +256,47 @@ def test_kernel_ids(input_data):
     for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
         assert itr["kernel_id"] in symbol_info.keys()
 
+    for itr in sdk_data["callback_records"]["kernel_dispatches"]:
+        assert itr["payload"]["kernel_id"] in symbol_info.keys()
+
+
+def test_kernel_dispatch_ids(input_data):
+    data = input_data
+    sdk_data = data["rocprofiler-sdk-json-tool"]
+
+    num_dispatches = len(sdk_data["buffer_records"]["kernel_dispatches"])
+    num_cb_dispatches = len(sdk_data["callback_records"]["kernel_dispatches"])
+
+    assert num_cb_dispatches == (2 * num_dispatches)
+
+    bf_seq_ids = []
+    for itr in sdk_data["buffer_records"]["kernel_dispatches"]:
+        bf_seq_ids.append(itr["dispatch_id"])
+
+    cb_seq_ids = []
+    for itr in sdk_data["callback_records"]["kernel_dispatches"]:
+        cb_seq_ids.append(itr["payload"]["dispatch_id"])
+
+    bf_seq_ids = sorted(bf_seq_ids)
+    cb_seq_ids = sorted(cb_seq_ids)
+
+    assert (2 * len(bf_seq_ids)) == len(cb_seq_ids)
+
+    assert bf_seq_ids[0] == cb_seq_ids[0]
+    assert bf_seq_ids[-1] == cb_seq_ids[-1]
+
+    def get_uniq(data):
+        return list(set(data))
+
+    bf_seq_ids_uniq = get_uniq(bf_seq_ids)
+    cb_seq_ids_uniq = get_uniq(cb_seq_ids)
+
+    assert bf_seq_ids == bf_seq_ids_uniq
+    assert len(cb_seq_ids) == (2 * len(cb_seq_ids_uniq))
+    assert len(bf_seq_ids) == num_dispatches
+    assert len(bf_seq_ids_uniq) == num_dispatches
+    assert len(cb_seq_ids_uniq) == num_dispatches
+
 
 def test_async_copy_direction(input_data):
     data = input_data