Put cached perfetto traces as default one (#2138)

* Put cached perfetto traces as default one * Improve cached data and perfetto traces in order to be more aligned with E2E tests * Addressing PR comments and findings * Force early instrumentation bundle instantiation * Sync-up insturumented containers with thread growth data * Revert ompvv number of host threads to default 8 * Fixed counter track namings for amd-smi * AIPROFSYST-34 [rocprof-sys] Update documentation describing newly introduced changes to default tracing mechanism
2025-12-22 12:47:35 +01:00
parent 7da3275b42
commit ba1380a75d
36 changed files with 557 additions and 183 deletions
@@ -103,6 +103,7 @@ extern "C"
    void rocprofsys_set_mpi_hidden(bool, bool) ROCPROFSYS_HIDDEN_API;
    void rocprofsys_push_trace_hidden(const char*) ROCPROFSYS_HIDDEN_API;
    void rocprofsys_pop_trace_hidden(const char*) ROCPROFSYS_HIDDEN_API;
+    void rocprofsys_flush_pending_region_cache_hidden() ROCPROFSYS_HIDDEN_API;
    void rocprofsys_push_region_hidden(const char*) ROCPROFSYS_HIDDEN_API;
    void rocprofsys_pop_region_hidden(const char*) ROCPROFSYS_HIDDEN_API;
    void rocprofsys_push_category_region_hidden(rocprofsys_category_t, const char*,
@@ -899,11 +899,12 @@ rocprofsys_finalize_hidden(void)
 #endif

    ROCPROFSYS_DEBUG_F("Stopping and destroying instrumentation bundles...\n");
-    for(size_t i = 0; i < thread_info::get_peak_num_threads(); ++i)
+    auto* _bundles = instrumentation_bundles::get();
+    for(size_t i = 0; _bundles && i < thread_info::get_peak_num_threads(); ++i)
    {
-        if(!instrumentation_bundles::get()) continue;
+        if(i >= _bundles->size()) continue;
        const auto& _info = thread_info::get(i, SequentTID);
-        auto&       itr   = instrumentation_bundles::get()->at(i);
+        auto&       itr   = _bundles->at(i);
        while(itr != nullptr && !itr->empty())
        {
            int _lvl = 1;
@@ -1026,6 +1027,11 @@ rocprofsys_finalize_hidden(void)

    tracing::copy_timemory_hash_ids();

+    // Flush any pending region cache entries (e.g., main entry point that wasn't
+    // explicitly stopped before finalization)
+    ROCPROFSYS_DEBUG_F("Flushing pending region cache entries...\n");
+    rocprofsys_flush_pending_region_cache_hidden();
+
    bool _perfetto_output_error = false;
    if(get_use_perfetto())
    {
@@ -120,6 +120,31 @@ cache_stop(const char* name)
                     rocprofsys::trait::name<CategoryT>::value);
    }
 }
+
+/// Flush all pending cached entries for this thread.
+/// Called during finalization to ensure entries that were started but not stopped
+/// (e.g., main entry point) are written to the trace cache.
+inline void
+flush_pending_cached_entries()
+{
+    const auto end_ts = static_cast<timestamp_t>(rocprofsys::comp::wall_clock::record());
+    uint64_t   thread_id = 0;
+
+    const auto& extended_info = rocprofsys::thread_info::get(std::this_thread::get_id());
+    if(extended_info.has_value() && extended_info->index_data.has_value())
+    {
+        constexpr size_t UNKNOWN_TIME = 0;
+        thread_id                     = extended_info->index_data->system_value;
+        rocprofsys::trace_cache::get_metadata_registry().add_thread_info(
+            { getppid(), getpid(), thread_id, UNKNOWN_TIME, UNKNOWN_TIME, "{}" });
+    }
+
+    for(const auto& [key, start_ts] : map_name_to_args)
+    {
+        cache_region(thread_id, key.name, start_ts, end_ts, key.category);
+    }
+    map_name_to_args.clear();
+}
 }  // namespace

 namespace tim
@@ -533,12 +533,22 @@ void
 cache_region(const rocprofiler_callback_tracing_record_t* record,
             const rocprofiler_timestamp_t                start_timestamp,
             const rocprofiler_timestamp_t end_timestamp, const std::string& call_stack,
-             const std::string& args_str, const std::string& category)
+             const std::string& args_str, const std::string& category,
+             std::string_view name = {})

 {
-    auto callback_tracing_info =
-        trace_cache::get_metadata_registry().get_callback_tracing_info();
-    auto _name = std::string{ callback_tracing_info.at(record->kind, record->operation) };
+    // Use provided name if available, otherwise fall back to API operation name
+    std::string _name;
+    if(name.empty())
+    {
+        auto callback_tracing_info =
+            trace_cache::get_metadata_registry().get_callback_tracing_info();
+        _name = std::string{ callback_tracing_info.at(record->kind, record->operation) };
+    }
+    else
+    {
+        _name = std::string{ name };
+    }

    trace_cache::get_buffer_storage().store(trace_cache::region_sample{
        record->thread_id, _name.c_str(), record->correlation_id.internal,
@@ -814,7 +824,7 @@ tool_tracing_callback_stop(
        cache_add_thread_info(record.thread_id);
        std::string args_str = get_args_string(args);
        cache_region(&record, _beg_ts, _end_ts, call_stack.dump(), args_str,
-                     trait::name<CategoryT>::value);
+                     trait::name<CategoryT>::value, _name);
    }
 }

@@ -31,6 +31,7 @@
 #include "core/state.hpp"
 #include "core/timemory.hpp"
 #include "core/utility.hpp"
+#include "library/thread_data_growth.hpp"
 #include "library/thread_deleter.hpp"

 #include <timemory/utility/macros.hpp>
@@ -54,15 +55,6 @@ using instrumentation_bundle_t =
 // allocator for instrumentation_bundle_t
 using bundle_allocator_t = tim::data::ring_buffer_allocator<instrumentation_bundle_t>;

-using grow_functor_t = int64_t (*)(int64_t);
-
-inline auto&
-grow_functors()
-{
-    static auto _v = container::stable_vector<grow_functor_t>{};
-    return _v;
-}
-
 template <typename Tp>
 struct base_thread_data
 {
@@ -77,7 +69,16 @@ struct base_thread_data
            }
            return (_v) ? _v->capacity() : 0;
        };
-        grow_functors().emplace_back(std::move(_func));
+        grow_functors().emplace_back(_func);
+
+        // Immediately sync this container to current peak_num_threads.
+        // This ensures containers instantiated after threads exceed
+        // max_supported_threads are properly sized.
+        auto _current_peak = get_current_peak_num_threads();
+        if(_current_peak > static_cast<int64_t>(max_supported_threads))
+        {
+            _func(_current_peak);
+        }
    }
 };

@@ -0,0 +1,63 @@
+// MIT License
+//
+// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All Rights Reserved.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all
+// copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+// SOFTWARE.
+
+#pragma once
+
+#include "core/concepts.hpp"
+#include "core/containers/stable_vector.hpp"
+
+#include <cstdint>
+#include <functional>
+
+namespace rocprofsys
+{
+using grow_functor_t = int64_t (*)(int64_t);
+
+inline auto&
+grow_functors()
+{
+    static auto _v = container::stable_vector<grow_functor_t>{};
+    return _v;
+}
+
+inline auto&
+get_peak_num_threads_callback()
+{
+    static std::function<int64_t()> _v = []() -> int64_t {
+        return static_cast<int64_t>(max_supported_threads);
+    };
+    return _v;
+}
+
+inline int64_t
+get_current_peak_num_threads()
+{
+    return get_peak_num_threads_callback()();
+}
+
+inline void
+set_peak_num_threads_callback(std::function<int64_t()> _cb)
+{
+    get_peak_num_threads_callback() = std::move(_cb);
+}
+
+}  // namespace rocprofsys
@@ -30,6 +30,7 @@
 #include "library/causal/delay.hpp"
 #include "library/runtime.hpp"
 #include "library/thread_data.hpp"
+#include "library/thread_data_growth.hpp"

 #include <timemory/backends/threading.hpp>
 #include <timemory/components/timing/backends.hpp>
@@ -111,6 +112,13 @@ init_index_data(int64_t _tid, bool _offset = false)
 thread_local int64_t offset_causal_count = 0;
 const auto           unknown_thread      = std::optional<thread_info>{};
 int64_t              peak_num_threads    = max_supported_threads;
+
+// Register callback to allow thread_data containers to query peak_num_threads
+// when they are instantiated, ensuring late-instantiated containers are properly sized.
+const auto peak_num_threads_callback_registered = []() {
+    set_peak_num_threads_callback([]() -> int64_t { return peak_num_threads; });
+    return true;
+}();
 }  // namespace

 std::string
@@ -124,6 +124,12 @@ rocprofsys_pop_trace_hidden(const char* name)
    rocprofsys::component::category_region<rocprofsys::category::host>::stop(name);
 }

+extern "C" void
+rocprofsys_flush_pending_region_cache_hidden()
+{
+    flush_pending_cached_entries();
+}
+
 //======================================================================================//
 ///
 ///