Stop all client contexts prior to finalization (#721)

* Stop all client contexts prior to finalization

* Update lib/common/container/static_vector.hpp

- improve emplace_back for non-{move,copy}-assignable object

* Update samples/intercept_table/client.cpp

- improve robustness against static object destruction

* Update lib/rocprofiler-sdk/context/context.cpp

- change storage of registered context array
  - stable_vector of optional contexts
  - common::static_object wrapper around stable_vector

* Update samples/intercept_table/client.cpp

- use variable template for underlying function pointer

[ROCm/rocprofiler-sdk commit: 939e23e9d1]

projects/rocprofiler-sdk/source/lib/common/container/static_vector.hpp

@@ -30,6 +30,7 @@
 #include <cstddef>
 #include <cstdlib>
 #include <initializer_list>
+#include <type_traits>
 
 namespace rocprofiler
 {
@@ -208,7 +209,7 @@ static_vector<Tp, N, AtomicSizeV>::emplace_back(Args&&... _v)
         else
             m_data[_idx] = Tp{std::forward<Args>(_v)...};
     }
-    else
+    else if constexpr(std::is_move_assignable<Tp>::value || std::is_copy_assignable<Tp>::value)
     {
         m_data[_idx] = {};
     }

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.cpp

@@ -50,9 +50,8 @@ namespace context
 {
 namespace
 {
-using reserve_size_t = common::container::reserve_size;
-using unique_context_vec_t =
-    common::container::stable_vector<allocator::unique_static_ptr_t<context>, 8>;
+using reserve_size_t       = common::container::reserve_size;
+using stable_context_vec_t = common::container::stable_vector<std::optional<context>, 8>;
 using active_context_vec_t = common::container::stable_vector<std::atomic<const context*>, 8>;
 
 constexpr auto invalid_client_idx = std::numeric_limits<uint32_t>::max();
@@ -83,10 +82,11 @@ get_client_index()
     return _v;
 }
 
-unique_context_vec_t&
+stable_context_vec_t*&
 get_registered_contexts_impl()
 {
-    static auto _v = unique_context_vec_t{reserve_size_t{unique_context_vec_t::chunk_size}};
+    static auto*& _v = common::static_object<stable_context_vec_t>::construct(
+        reserve_size_t{stable_context_vec_t::chunk_size});
     return _v;
 }
 
@@ -100,14 +100,7 @@ get_num_active_contexts()
 active_context_vec_t&
 get_active_contexts_impl()
 {
-    static auto* _v    = new active_context_vec_t{reserve_size_t{active_context_vec_t::chunk_size}};
-    static auto  _once = std::once_flag{};
-    std::call_once(_once, std::atexit, []() {
-        for(auto& itr : *_v)
-        {
-            itr.store(nullptr);
-        }
-    });
+    static auto* _v = new active_context_vec_t{reserve_size_t{active_context_vec_t::chunk_size}};
     return *_v;
 }
 
@@ -243,13 +236,16 @@ context_array_t&
 get_registered_contexts(context_array_t& data, context_filter_t filter)
 {
     data.clear();
-    auto num_ctx = get_registered_contexts_impl().size();
+
+    if(!get_registered_contexts_impl()) return data;
+
+    auto num_ctx = get_registered_contexts_impl()->size();
     if(num_ctx <= 0) return data;
 
     data.reserve(num_ctx);
-    for(auto& itr : get_registered_contexts_impl())
+    for(auto& itr : *get_registered_contexts_impl())
     {
-        const auto* ctx = itr.get();
+        const auto* ctx = &itr.value();
         if(ctx)
         {
             if(!filter || (filter && filter(ctx))) data.emplace_back(ctx);
@@ -332,15 +328,15 @@ allocate_context()
     auto _lk = std::unique_lock<std::mutex>{get_contexts_mutex()};
 
     // initial context identifier number
-    auto _idx = get_registered_contexts_impl().size() + get_contexts_offset();
+    auto _idx = get_registered_contexts_impl()->size() + get_contexts_offset();
 
     // make space in registered
-    get_registered_contexts_impl().emplace_back(nullptr);
+    get_registered_contexts_impl()->emplace_back();
 
     // create an entry in the registered
-    auto& _cfg_v = get_registered_contexts_impl().back();
-    _cfg_v       = allocator::make_unique_static<context>();
-    auto* _cfg   = _cfg_v.get();
+    auto& _cfg_v = get_registered_contexts_impl()->back();
+    _cfg_v.emplace();
+    auto* _cfg = &_cfg_v.value();
     // ...
 
     if(!_cfg) return std::nullopt;
@@ -361,8 +357,8 @@ get_mutable_registered_context(rocprofiler_context_id_t id)
 {
     if(id.handle < get_contexts_offset()) return nullptr;
     auto _idx = id.handle - get_contexts_offset();
-    if(_idx >= get_registered_contexts_impl().size()) return nullptr;
-    return get_registered_contexts_impl().at(_idx).get();
+    if(_idx >= get_registered_contexts_impl()->size()) return nullptr;
+    return &get_registered_contexts_impl()->at(_idx).value();
 }
 
 const context*
@@ -382,7 +378,9 @@ start_context(rocprofiler_context_id_t context_id)
 {
     if(context_id.handle < get_contexts_offset()) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND;
 
-    if((context_id.handle - get_contexts_offset()) >= get_registered_contexts_impl().size())
+    if(!get_registered_contexts_impl()) return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND;
+
+    if((context_id.handle - get_contexts_offset()) >= get_registered_contexts_impl()->size())
         return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND;
 
     const auto* cfg = get_registered_context(context_id);
@@ -406,7 +404,7 @@ start_context(rocprofiler_context_id_t context_id)
         }
     }
 
-    uint64_t rocp_tot_contexts = get_registered_contexts_impl().size();
+    uint64_t rocp_tot_contexts = get_registered_contexts_impl()->size();
     auto     idx               = rocp_tot_contexts;
     auto&    active_contexts   = get_active_contexts_impl();
     {
@@ -482,6 +480,25 @@ stop_context(rocprofiler_context_id_t idx)
     return ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND;  // compare exchange failed
 }
 
+rocprofiler_status_t
+stop_client_contexts(rocprofiler_client_id_t client_id)
+{
+    if(!get_registered_contexts_impl()) return ROCPROFILER_STATUS_ERROR;
+
+    auto ret = ROCPROFILER_STATUS_SUCCESS;
+    for(auto& itr : *get_registered_contexts_impl())
+    {
+        if(itr->client_idx == client_id.handle)
+        {
+            auto status = stop_context(rocprofiler_context_id_t{itr->context_idx});
+            if(status != ROCPROFILER_STATUS_SUCCESS &&
+               status != ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND)
+                ret = status;
+        }
+    }
+    return ret;
+}
+
 void
 deactivate_client_contexts(rocprofiler_client_id_t client_id)
 {
@@ -498,7 +515,7 @@ deactivate_client_contexts(rocprofiler_client_id_t client_id)
 void
 deregister_client_contexts(rocprofiler_client_id_t client_id)
 {
-    for(auto& itr : get_registered_contexts_impl())
+    for(auto& itr : *get_registered_contexts_impl())
     {
         if(itr->client_idx == client_id.handle && buffer::get_buffers())
         {

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/context/context.hpp

@@ -182,8 +182,9 @@ validate_context(const context* cfg);
 rocprofiler_status_t
 start_context(rocprofiler_context_id_t id);
 
-/// \brief disable the contexturation.
-rocprofiler_status_t stop_context(rocprofiler_context_id_t);
+/// \brief remove context from active array
+rocprofiler_status_t
+stop_context(rocprofiler_context_id_t id);
 
 using context_array_t = common::container::small_vector<const context*>;
 
@@ -216,10 +217,16 @@ get_active_contexts(context_array_t& data, context_filter_t filter = default_con
 context_array_t
 get_active_contexts(context_filter_t filter = default_context_filter);
 
-void deactivate_client_contexts(rocprofiler_client_id_t);
+/// \brief disable the contexturation.
+rocprofiler_status_t
+stop_client_contexts(rocprofiler_client_id_t id);
+
+void
+deactivate_client_contexts(rocprofiler_client_id_t id);
 
 // should only be called if the client failed to initialize
-void deregister_client_contexts(rocprofiler_client_id_t);
+void
+deregister_client_contexts(rocprofiler_client_id_t id);
 
 inline bool
 default_context_filter(const context* val)

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/registration.cpp

@@ -471,6 +471,7 @@ invoke_client_finalizer(rocprofiler_client_id_t client_id)
         if(itr && itr->internal_client_id.handle == client_id.handle &&
            itr->mutable_client_id.handle == client_id.handle)
         {
+            context::stop_client_contexts(itr->internal_client_id);
             if(itr->configure_result && itr->configure_result->finalize)
             {
                 // set to nullptr so finalize only gets called once

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/tests/external_correlation.cpp

@@ -276,8 +276,10 @@ TEST(rocprofiler_lib, callback_external_correlation)
 
     static fini_func_t tool_fini = [](void* client_data) -> void {
         auto* cb_data = static_cast<callback_data*>(client_data);
-        ROCPROFILER_CALL(rocprofiler_stop_context(cb_data->client_ctx),
-                         "rocprofiler context stop failed");
+        int   status  = 0;
+        ROCPROFILER_CALL(rocprofiler_context_is_active(cb_data->client_ctx, &status),
+                         "rocprofiler_context_is_active failed");
+        EXPECT_EQ(status, 0);
 
         static_cast<callback_data*>(client_data)->client_workflow_count++;
     };
@@ -432,8 +434,10 @@ TEST(rocprofiler_lib, buffered_external_correlation)
 
     static fini_func_t tool_fini = [](void* client_data) -> void {
         auto* cb_data = static_cast<callback_data*>(client_data);
-        ROCPROFILER_CALL(rocprofiler_stop_context(cb_data->client_ctx),
-                         "rocprofiler context stop failed");
+        int   status  = 0;
+        ROCPROFILER_CALL(rocprofiler_context_is_active(cb_data->client_ctx, &status),
+                         "rocprofiler_context_is_active failed");
+        EXPECT_EQ(status, 0);
 
         static_cast<callback_data*>(client_data)->client_workflow_count++;
     };

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/tests/intercept_table.cpp

@@ -218,8 +218,10 @@ TEST(rocprofiler_lib, intercept_table_and_callback_tracing)
 
     static fini_func_t tool_fini = [](void* client_data) -> void {
         auto* cb_data = static_cast<callback_data_ext*>(client_data);
-        ROCPROFILER_CALL(rocprofiler_stop_context(cb_data->client_hsa_ctx),
-                         "rocprofiler context stop failed");
+        int   status  = 0;
+        ROCPROFILER_CALL(rocprofiler_context_is_active(cb_data->client_hsa_ctx, &status),
+                         "rocprofiler_context_is_active failed");
+        EXPECT_EQ(status, 0);
 
         static_cast<callback_data_ext*>(client_data)->client_workflow_count++;
     };
@@ -375,9 +377,10 @@ TEST(rocprofiler_lib, intercept_table_and_callback_tracing_disable_context)
 
     static fini_func_t tool_fini = [](void* client_data) -> void {
         auto* cb_data = static_cast<callback_data_ext*>(client_data);
-        ROCPROFILER_CALL_EXPECT(rocprofiler_stop_context(cb_data->client_hsa_ctx),
-                                "rocprofiler context stop",
-                                ROCPROFILER_STATUS_ERROR_CONTEXT_NOT_FOUND);
+        int   status  = 0;
+        ROCPROFILER_CALL(rocprofiler_context_is_active(cb_data->client_hsa_ctx, &status),
+                         "rocprofiler_context_is_active failed");
+        EXPECT_EQ(status, 0);
 
         ROCPROFILER_CALL_EXPECT(rocprofiler_start_context(cb_data->client_hsa_ctx),
                                 "rocprofiler context start",

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/tests/registration.cpp

@@ -278,8 +278,10 @@ TEST(rocprofiler_lib, callback_registration_lambda_with_result)
 
     static fini_func_t tool_fini = [](void* client_data) -> void {
         auto* cb_data = static_cast<callback_data*>(client_data);
-        ROCPROFILER_CALL(rocprofiler_stop_context(cb_data->client_ctx),
-                         "rocprofiler context stop failed");
+        int   status  = 0;
+        ROCPROFILER_CALL(rocprofiler_context_is_active(cb_data->client_ctx, &status),
+                         "rocprofiler_context_is_active failed");
+        EXPECT_EQ(status, 0);
 
         static_cast<callback_data*>(client_data)->client_workflow_count++;
     };
@@ -429,8 +431,10 @@ TEST(rocprofiler_lib, buffer_registration_lambda_with_result)
         auto* cb_data = static_cast<callback_data*>(client_data);
         ROCPROFILER_CALL(rocprofiler_flush_buffer(cb_data->client_buffer),
                          "rocprofiler context stop failed");
-        ROCPROFILER_CALL(rocprofiler_stop_context(cb_data->client_ctx),
-                         "rocprofiler context stop failed");
+        int status = 0;
+        ROCPROFILER_CALL(rocprofiler_context_is_active(cb_data->client_ctx, &status),
+                         "rocprofiler_context_is_active failed");
+        EXPECT_EQ(status, 0);
 
         static_cast<callback_data*>(client_data)->client_workflow_count++;
     };

projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/tests/roctx.cpp

@@ -360,8 +360,10 @@ TEST(rocprofiler_lib, roctx_callback_tracing)
 
     static fini_func_t tool_fini = [](void* client_data) -> void {
         auto* cb_data = static_cast<callback_data*>(client_data);
-        ROCPROFILER_CALL(rocprofiler_stop_context(cb_data->client_ctx),
-                         "rocprofiler context stop failed");
+        int   status  = 0;
+        ROCPROFILER_CALL(rocprofiler_context_is_active(cb_data->client_ctx, &status),
+                         "rocprofiler_context_is_active failed");
+        EXPECT_EQ(status, 0);
 
         static_cast<callback_data*>(client_data)->client_workflow_count++;
     };
@@ -489,8 +491,10 @@ TEST(rocprofiler_lib, roctx_buffered_tracing)
         auto* cb_data = static_cast<callback_data*>(client_data);
         ROCPROFILER_CALL(rocprofiler_flush_buffer(cb_data->client_buffer),
                          "rocprofiler context stop failed");
-        ROCPROFILER_CALL(rocprofiler_stop_context(cb_data->client_ctx),
-                         "rocprofiler context stop failed");
+        int status = 0;
+        ROCPROFILER_CALL(rocprofiler_context_is_active(cb_data->client_ctx, &status),
+                         "rocprofiler_context_is_active failed");
+        EXPECT_EQ(status, 0);
 
         static_cast<callback_data*>(client_data)->client_workflow_count++;
     };