Allow events to be grouped by HIP stream ID (#274)

- Corelate memory_copy and kernel_dispatch events with their HIP stream_id and add stream_id as an annotation in Perfetto.
- By default, group memory_copy and kernel_dispatch events in Perfetto output by their stream_id.
- Add option, with the configuration setting ROCPROFSYS_ROCM_GROUP_BY_QUEUE, to group by HSA queue instead.

---------

Signed-off-by: David Galiffi <David.Galiffi@amd.com>
Co-authored-by: David Galiffi <David.Galiffi@amd.com>

[ROCm/rocprofiler-systems commit: 4b4a846b58]

projects/rocprofiler-systems/source/lib/core/categories.hpp

@@ -96,6 +96,7 @@ ROCPROFSYS_DEFINE_CATEGORY(category, rocm_hip_api, ROCPROFSYS_CATEGORY_ROCM_HIP_
 ROCPROFSYS_DEFINE_CATEGORY(category, rocm_hsa_api, ROCPROFSYS_CATEGORY_ROCM_HSA_API, "rocm_hsa_api", "ROCm HSA functions")
 ROCPROFSYS_DEFINE_CATEGORY(category, rocm_kernel_dispatch, ROCPROFSYS_CATEGORY_ROCM_KERNEL_DISPATCH, "rocm_kernel_dispatch", "ROCm Kernel dispatch")
 ROCPROFSYS_DEFINE_CATEGORY(category, rocm_memory_copy, ROCPROFSYS_CATEGORY_ROCM_MEMORY_COPY, "rocm_memory_copy", "ROCm Async Memory Copy")
+ROCPROFSYS_DEFINE_CATEGORY(category, rocm_hip_stream, ROCPROFSYS_CATEGORY_ROCM_HIP_STREAM, "rocm_hip_stream", "ROCm HIP Stream")
 ROCPROFSYS_DEFINE_CATEGORY(category, rocm_scratch_memory, ROCPROFSYS_CATEGORY_ROCM_SCRATCH_MEMORY, "rocm_scratch_memory", "ROCm kernel scratch memory reallocations")
 ROCPROFSYS_DEFINE_CATEGORY(category, rocm_page_migration, ROCPROFSYS_CATEGORY_ROCM_PAGE_MIGRATION, "rocm_page_migration", "ROCm memory page migration")
 ROCPROFSYS_DEFINE_CATEGORY(category, rocm_counter_collection, ROCPROFSYS_CATEGORY_ROCM_COUNTER_COLLECTION, "rocm_counter_collection", "ROCm device counter collection")
@@ -166,6 +167,7 @@ using name = perfetto_category<Tp...>;
         ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_hsa_api),                            \
         ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_kernel_dispatch),                    \
         ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_memory_copy),                        \
+        ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_hip_stream),                         \
         ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_scratch_memory),                     \
         ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_page_migration),                     \
         ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_counter_collection),                 \

projects/rocprofiler-systems/source/lib/core/rocprofiler-sdk.cpp

@@ -372,6 +372,22 @@ config_settings(const std::shared_ptr<settings>& _config)
 
     for(const auto& itr : buffered_tracing_info)
         _add_operation_settings(itr.name, itr, buffered_operation_option_names);
+
+    // Add the ROCPROFSYS_ROCM_GROUP_BY_QUEUE setting if the hip_stream domain is present
+    // in supported ROCProfiler-SDK domains.
+    auto _has_hip_stream = std::find(_domain_choices.begin(), _domain_choices.end(),
+                                     "hip_stream") != _domain_choices.end();
+
+    if(_has_hip_stream)
+    {
+        ROCPROFSYS_CONFIG_SETTING(
+            bool, "ROCPROFSYS_ROCM_GROUP_BY_QUEUE",
+            "By default, Perfetto trace will show the HIP streams to which kernel "
+            "and memory copy operations submitted. With the "
+            "`ROCPROFSYS_ROCM_GROUP_BY_QUEUE` option, the trace will display HSA queues "
+            "to which these kernel and memory operations were submitted.",
+            false, "rocm", "perfetto");
+    }
 }
 
 std::unordered_set<rocprofiler_callback_tracing_kind_t>
@@ -550,6 +566,16 @@ get_rocm_events()
         " ,;\t\n");
 }
 
+bool
+get_group_by_queue(void)
+{
+    std::optional<bool> _group_by_queue =
+        config::get_setting_value<bool>("ROCPROFSYS_ROCM_GROUP_BY_QUEUE");
+    bool _ret = _group_by_queue.value_or(true);
+
+    return _ret;
+}
+
 std::vector<int32_t>
 get_operations(rocprofiler_callback_tracing_kind_t kindv)
 {

projects/rocprofiler-systems/source/lib/core/rocprofiler-sdk.hpp

@@ -70,6 +70,9 @@ get_operations(rocprofiler_buffer_tracing_kind_t kindv);
 std::vector<std::string>
 get_rocm_events();
 
+bool
+get_group_by_queue();
+
 std::unordered_set<int32_t>
 get_backtrace_operations(rocprofiler_callback_tracing_kind_t kindv);