Use rocprofiler-sdk for RCCL-API tracing (#126)

- Add support for RCCL API tracing through rocprofiler-sdk. - Refactored the comm_data code to use the SDK RCCL_API callbacks. - Add a runtime version check for SDK to gate callback enablement, rather than just the compile-time check. - Fixed: SAMPLING_TIMEOUT was not being handled correctly in add_test. [ROCm/rocprofiler-systems commit: af77d93f75]
2025-06-06 11:36:17 -04:00
@@ -35,7 +35,6 @@ set(core_headers
    ${CMAKE_CURRENT_LIST_DIR}/mproc.hpp
    ${CMAKE_CURRENT_LIST_DIR}/perf.hpp
    ${CMAKE_CURRENT_LIST_DIR}/perfetto.hpp
-    ${CMAKE_CURRENT_LIST_DIR}/rccl.hpp
    ${CMAKE_CURRENT_LIST_DIR}/redirect.hpp
    ${CMAKE_CURRENT_LIST_DIR}/rocprofiler-sdk.hpp
    ${CMAKE_CURRENT_LIST_DIR}/state.hpp
@@ -562,10 +562,9 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
        _data.processed_environs.emplace("periods");
    }

-    strset_t _backend_choices = {
-        "all",     "kokkosp",         "mpip",        "ompt",       "rcclp",
-        "amd-smi", "rocprofiler-sdk", "mutex-locks", "spin-locks", "rw-locks"
-    };
+    strset_t _backend_choices = { "all",        "kokkosp", "mpip", "ompt",
+                                  "rcclp",      "amd-smi", "rocm", "mutex-locks",
+                                  "spin-locks", "rw-locks" };

 #if !defined(ROCPROFSYS_USE_MPI) && !defined(ROCPROFSYS_USE_MPI_HEADERS)
    _backend_choices.erase("mpip");
@@ -575,14 +574,10 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
    _backend_choices.erase("ompt");
 #endif

-#if !defined(ROCPROFSYS_USE_RCCL)
-    _backend_choices.erase("rcclp");
-#endif
-
 #if !defined(ROCPROFSYS_USE_ROCM)
    _backend_choices.erase("amd-smi");
-    _backend_choices.erase("rocprofiler-sdk");
    _backend_choices.erase("rocm");
+    _backend_choices.erase("rcclp");
 #endif

    if(gpu::device_count() == 0)
@@ -590,13 +585,8 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
        // remove GPU-specific backends
        _backend_choices.erase("rcclp");
        _backend_choices.erase("amd-smi");
-        _backend_choices.erase("rocprofiler-sdk");
        _backend_choices.erase("rocm");

-#if defined(ROCPROFSYS_USE_RCCL)
-        update_env(_data, "ROCPROFSYS_USE_RCCLP", false);
-#endif
-
 #if defined(ROCPROFSYS_USE_ROCM)
        update_env(_data, "ROCPROFSYS_USE_AMD_SMI", false);
        update_env(_data, "ROCPROFSYS_USE_ROCM", false);
@@ -102,6 +102,7 @@ ROCPROFSYS_DEFINE_CATEGORY(category, rocm_counter_collection, ROCPROFSYS_CATEGOR
 ROCPROFSYS_DEFINE_CATEGORY(category, rocm_marker_api, ROCPROFSYS_CATEGORY_ROCM_MARKER_API, "rocm_marker_api", "ROCTx labels")
 ROCPROFSYS_DEFINE_CATEGORY(category, rocm_rocdecode_api, ROCPROFSYS_CATEGORY_ROCM_ROCDECODE_API, "rocm_rocdecode_api", "ROCm RocDecode API")
 ROCPROFSYS_DEFINE_CATEGORY(category, rocm_rocjpeg_api, ROCPROFSYS_CATEGORY_ROCM_ROCJPEG_API, "rocm_rocjpeg_api", "ROCm RocJPEG API")
+ROCPROFSYS_DEFINE_CATEGORY(category, rocm_rccl_api, ROCPROFSYS_CATEGORY_ROCM_RCCL_API, "rocm_rccl_api", "ROCm RCCL API")
 ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi, ROCPROFSYS_CATEGORY_AMD_SMI, "amd_smi", "AMD-SMI data")
 ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_gfx_busy, ROCPROFSYS_CATEGORY_AMD_SMI_BUSY_GFX, "device_busy_gfx", "Busy percentage of GFX engine on a GPU device")
 ROCPROFSYS_DEFINE_CATEGORY(category, amd_smi_umc_busy, ROCPROFSYS_CATEGORY_AMD_SMI_BUSY_UMC, "device_busy_umc", "Busy percentage of UMC engin on a GPU device")
@@ -171,6 +172,7 @@ using name = perfetto_category<Tp...>;
        ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_marker_api),                         \
        ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_rocdecode_api),                      \
        ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_rocjpeg_api),                        \
+        ROCPROFSYS_PERFETTO_CATEGORY(category::rocm_rccl_api),                           \
        ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi),                                 \
        ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_gfx_busy),                        \
        ROCPROFSYS_PERFETTO_CATEGORY(category::amd_smi_umc_busy),                        \
@@ -1359,6 +1359,14 @@ configure_disabled_settings(const std::shared_ptr<settings>& _config)
    _config->find("ROCPROFSYS_USE_AMD_SMI")->second->set_hidden(true);
    for(const auto& itr : _config->disable_category("amd_smi"))
        _config->find(itr)->second->set_hidden(true);
+
+    _config->find("ROCPROFSYS_USE_RCCLP")->second->set_hidden(true);
+    for(const auto& itr : _config->disable_category("rcclp"))
+        _config->find(itr)->second->set_hidden(true);
+
+    _config->find("ROCPROFSYS_USE_ROCM")->second->set_hidden(true);
+    for(const auto& itr : _config->disable_category("rocm"))
+        _config->find(itr)->second->set_hidden(true);
 #endif

 #if defined(ROCPROFSYS_USE_OMPT) || ROCPROFSYS_USE_OMPT == 0
@@ -1,29 +0,0 @@
-// MIT License
-//
-// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All Rights Reserved.
-//
-// Permission is hereby granted, free of charge, to any person obtaining a copy
-// of this software and associated documentation files (the "Software"), to deal
-// in the Software without restriction, including without limitation the rights
-// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-// copies of the Software, and to permit persons to whom the Software is
-// furnished to do so, subject to the following conditions:
-//
-// The above copyright notice and this permission notice shall be included in all
-// copies or substantial portions of the Software.
-//
-// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-// SOFTWARE.
-
-#pragma once
-
-#include "core/defines.hpp"
-
-#if defined(ROCPROFSYS_USE_RCCL) && ROCPROFSYS_USE_RCCL > 0
-#    include <rccl/rccl.h>
-#endif
@@ -218,6 +218,30 @@ get_operations_impl(const std::unordered_set<int32_t>& _complete,

 }  // namespace

+/// @brief Return the version of the rocprofiler-sdk
+/// @return The version of the rocprofiler-sdk or 0 if not initialized
+version_info&
+get_version()
+{
+    static auto _version = version_info{ 0 };
+
+    if(_version.formatted == 0)
+    {
+        uint32_t _major = 0;
+        uint32_t _minor = 0;
+        uint32_t _patch = 0;
+
+        ROCPROFILER_CALL(rocprofiler_get_version(&_major, &_minor, &_patch));
+
+        _version.major     = _major;
+        _version.minor     = _minor;
+        _version.patch     = _patch;
+        _version.formatted = _major * 10000 + _minor * 100 + _patch;
+    }
+
+    return _version;
+}
+
 void
 config_settings(const std::shared_ptr<settings>& _config)
 {
@@ -319,6 +343,7 @@ config_settings(const std::shared_ptr<settings>& _config)
             join::join(join::array_config{ ", ", "", "" }, _domain_choices));
    auto _domain_defaults = std::string{ "hip_runtime_api,marker_api,kernel_dispatch,"
                                         "memory_copy,scratch_memory" };
+
 #    if(ROCPROFILER_VERSION < 10000)
    _domain_defaults.append(",page_migration");
 #    endif
@@ -353,28 +378,48 @@ std::unordered_set<rocprofiler_callback_tracing_kind_t>
 get_callback_domains()
 {
    const auto callback_tracing_info = rocprofiler::sdk::get_callback_tracing_names();
-    const auto supported = std::unordered_set<rocprofiler_callback_tracing_kind_t>
-    {
+    auto       supported = std::unordered_set<rocprofiler_callback_tracing_kind_t>{
        ROCPROFILER_CALLBACK_TRACING_HSA_CORE_API,
-            ROCPROFILER_CALLBACK_TRACING_HSA_AMD_EXT_API,
-            ROCPROFILER_CALLBACK_TRACING_HSA_IMAGE_EXT_API,
-            ROCPROFILER_CALLBACK_TRACING_HSA_FINALIZE_EXT_API,
-            ROCPROFILER_CALLBACK_TRACING_HIP_RUNTIME_API,
-            ROCPROFILER_CALLBACK_TRACING_HIP_COMPILER_API,
-            ROCPROFILER_CALLBACK_TRACING_MARKER_CORE_API,
-            ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
-#    if(ROCPROFILER_VERSION >= 700)
-            ROCPROFILER_CALLBACK_TRACING_ROCDECODE_API,
-            ROCPROFILER_CALLBACK_TRACING_ROCJPEG_API,
-#    endif
+        ROCPROFILER_CALLBACK_TRACING_HSA_AMD_EXT_API,
+        ROCPROFILER_CALLBACK_TRACING_HSA_IMAGE_EXT_API,
+        ROCPROFILER_CALLBACK_TRACING_HSA_FINALIZE_EXT_API,
+        ROCPROFILER_CALLBACK_TRACING_HIP_RUNTIME_API,
+        ROCPROFILER_CALLBACK_TRACING_HIP_COMPILER_API,
+        ROCPROFILER_CALLBACK_TRACING_MARKER_CORE_API,
+        ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
    };

+    auto _version = get_version();
+    ROCPROFSYS_WARNING_IF(_version.formatted == 0,
+                          "Warning! rocprofiler-sdk version not initialized\n");
+
+#    if(ROCPROFILER_VERSION >= 600)
+    if(_version.formatted >= 600)
+    {
+        // Argument tracing is supported in rocprofiler-sdk 0.6.0 and later
+        supported.emplace(ROCPROFILER_CALLBACK_TRACING_RCCL_API);
+        supported.emplace(ROCPROFILER_CALLBACK_TRACING_ROCDECODE_API);
+    }
+#    endif
+#    if(ROCPROFILER_VERSION >= 700)
+    if(_version.formatted >= 700)
+    {
+        supported.emplace(ROCPROFILER_CALLBACK_TRACING_ROCJPEG_API);
+    }
+#    endif
+
    auto _data = std::unordered_set<rocprofiler_callback_tracing_kind_t>{};
    auto _domains =
        tim::delimit(config::get_setting_value<std::string>("ROCPROFSYS_ROCM_DOMAINS")
                         .value_or(std::string{}),
                     " ,;:\t\n");

+    if(config::get_use_rcclp() && _version.formatted >= 600)
+    {
+        // Translate ROCPROFSYS_USE_RCCLP to entry in ROCPROFSYS_ROCM_DOMAINS
+        _data.emplace(ROCPROFILER_CALLBACK_TRACING_RCCL_API);
+    }
+
    const auto valid_choices =
        settings::instance()->at("ROCPROFSYS_ROCM_DOMAINS")->get_choices();

@@ -582,6 +627,12 @@ void
 config_settings(const std::shared_ptr<settings>&)
 {}

+version_info&
+get_version()
+{
+    static auto _version = version_info{ 0 };
+    return _version;
+}
 }  // namespace rocprofiler_sdk
 }  // namespace rocprofsys

@@ -39,9 +39,20 @@ namespace rocprofsys
 {
 namespace rocprofiler_sdk
 {
+struct version_info
+{
+    uint32_t major     = 0;
+    uint32_t minor     = 0;
+    uint32_t patch     = 0;
+    uint32_t formatted = 0;  // major * 10000 + minor * 100 + patch
+};
+
 void
 config_settings(const std::shared_ptr<settings>&);

+version_info&
+get_version();
+
 #if defined(ROCPROFSYS_USE_ROCM)

 std::unordered_set<rocprofiler_callback_tracing_kind_t>