SWDEV-445711: Workaround for race condition on exit

Change-Id: I7391637005ccc7ea89611f28691c02e569197dff

src/core/hsa/queues/queue.cpp

@@ -384,25 +384,27 @@ void SignalAsyncReadyHandler(const hsa_signal_t& signal, void* data) {
           signal, HSA_SIGNAL_CONDITION_EQ, 0, AsyncSignalReadyHandler, data);
   if (status != HSA_STATUS_SUCCESS) fatal("hsa_amd_signal_async_handler failed");
 }
-bool AsyncSignalHandler(hsa_signal_value_t signal_value, void* data) {
+bool AsyncSignalHandler(hsa_signal_value_t signal_value, void* data)
+{
   auto queue_info_session = static_cast<queue_info_session_t*>(data);
+  if (!queue_info_session) return true;
+
   rocprofiler::ROCProfiler_Singleton& rocprofiler_singleton =
-      rocprofiler::ROCProfiler_Singleton::GetInstance();
+                                          rocprofiler::ROCProfiler_Singleton::GetInstance();
   rocprofiler::HSASupport_Singleton& hsasupport_singleton =
-      rocprofiler::HSASupport_Singleton::GetInstance();
-  if (!queue_info_session || !rocprofiler_singleton.GetSession(queue_info_session->session_id) ||
-      !rocprofiler_singleton.GetSession(queue_info_session->session_id)->GetProfiler())
-    return true;
+                                          rocprofiler::HSASupport_Singleton::GetInstance();
+
   rocprofiler::Session* session = rocprofiler_singleton.GetSession(queue_info_session->session_id);
+  if (!session) return true;
+
   std::lock_guard<std::mutex> lock(session->GetSessionLock());
   rocprofiler::profiler::Profiler* profiler = session->GetProfiler();
-  std::vector<pending_signal_t*> pending_signals = const_cast<std::vector<pending_signal_t*>&>(
-      profiler->GetPendingSignals(queue_info_session->writer_id));
+  if (!profiler) return true;
 
-  if (!pending_signals.empty()) {
-    for (auto it = pending_signals.begin(); it != pending_signals.end();
-         it = pending_signals.erase(it)) {
-      auto& pending = *it;
+  auto pending_signals = profiler->MovePendingSignals(queue_info_session->writer_id);
+
+  for (auto& pending : pending_signals)
+  {
       if (hsasupport_singleton.GetCoreApiTable().hsa_signal_load_relaxed_fn(pending->new_signal))
        return true;
       hsa_amd_profiling_dispatch_time_t time;
@@ -458,7 +460,7 @@ bool AsyncSignalHandler(hsa_signal_value_t signal_value, void* data) {
             rocprofiler::metrics::GetMetricsData(pending->context->results_map,
                                                  pending->context->metrics_list,
                                                  time.end - time.start);
-          AddRecordCounters(&record, pending);
+          AddRecordCounters(&record, pending.get());
         } else {
           if (session->FindBuffer(pending->buffer_id)) {
             Memory::GenericBuffer* buffer = session->GetBuffer(pending->buffer_id);
@@ -503,14 +505,11 @@ bool AsyncSignalHandler(hsa_signal_value_t signal_value, void* data) {
 
       }
 
-    
-
       if (pending->new_signal.handle)
        hsasupport_singleton.GetCoreApiTable().hsa_signal_destroy_fn(pending->new_signal);
       if (queue_info_session->interrupt_signal.handle)
         hsasupport_singleton.GetCoreApiTable().hsa_signal_destroy_fn(
             queue_info_session->interrupt_signal);
-    }
   }
   delete queue_info_session;
   ACTIVE_INTERRUPT_SIGNAL_COUNT.fetch_sub(1, std::memory_order_relaxed);
@@ -529,7 +528,8 @@ void CreateSignal(uint32_t attribute, hsa_signal_t* signal) {
   HSASupport_Singleton::GetInstance().CreateSignal(attribute, signal);
 }
 
-rocprofiler_session_id_t session_id = rocprofiler_session_id_t{0};
+rocprofiler_session_id_t Queue::session_id = rocprofiler_session_id_t{0};
+std::shared_mutex Queue::session_id_mutex;
 // Counter Names declaration
 std::vector<std::string> session_data;
 
@@ -546,9 +546,13 @@ uint32_t replay_mode_count = 0;
 
 rocprofiler::Session* session = nullptr;
 
-void ResetSessionID(rocprofiler_session_id_t id) { session_id = id; }
+void Queue::ResetSessionID(rocprofiler_session_id_t id)
+{
+  std::unique_lock<std::shared_mutex> session_id_lock(session_id_mutex);
+  session_id = id;
+}
 
-void CheckNeededProfileConfigs() {
+void Queue::CheckNeededProfileConfigs() {
   rocprofiler_session_id_t internal_session_id;
   // Getting Session ID
   rocprofiler::ROCProfiler_Singleton& rocprofiler_singleton =
@@ -609,7 +613,9 @@ std::atomic<uint32_t> WRITER_ID{0};
  * interceptor by invoking the writer function.
  */
 void Queue::WriteInterceptor(const void* packets, uint64_t pkt_count, uint64_t user_pkt_index,
-                             void* data, hsa_amd_queue_intercept_packet_writer writer) {
+                             void* data, hsa_amd_queue_intercept_packet_writer writer)
+{
+  std::shared_lock<std::shared_mutex> session_id_lock(session_id_mutex);
   const Packet::packet_t* packets_arr = reinterpret_cast<const Packet::packet_t*>(packets);
   std::vector<Packet::packet_t> transformed_packets;
 
@@ -669,7 +675,6 @@ void Queue::WriteInterceptor(const void* packets, uint64_t pkt_count, uint64_t u
       Packet::CreateBarrierPacket(&transformed_packets, &block_signal, &block_signal);
       }
 
-
       uint32_t writer_id = WRITER_ID.fetch_add(1, std::memory_order_release);
 
       if (session_data_count > 0 && is_counter_collection_mode && profiles.size() > 0 &&

src/core/hsa/queues/queue.h

@@ -35,6 +35,7 @@
 #include <string>
 #include <vector>
 #include <condition_variable>
+#include <shared_mutex>
 #include "src/core/session/profiler/profiler.h"
 
 namespace rocprofiler {
@@ -87,7 +88,11 @@ class Queue {
   hsa_signal_t GetReadySignal();
   hsa_signal_t GetBlockSignal();
 
+  static void ResetSessionID(rocprofiler_session_id_t id = rocprofiler_session_id_t{0});
+  static void CheckNeededProfileConfigs();
  private:
+  static std::shared_mutex session_id_mutex;
+  static rocprofiler_session_id_t session_id;
 
   hsa_agent_t cpu_agent_;
   hsa_agent_t gpu_agent_;
@@ -113,8 +118,6 @@ struct queue_info_session_t {
 
 void AddRecordCounters(rocprofiler_record_profiler_t* record, const pending_signal_t& pending);
 
-void ResetSessionID(rocprofiler_session_id_t id = rocprofiler_session_id_t{0});
-
 void CheckPacketReqiurements();
 
 }  // namespace queue

src/core/session/att/att.cpp

@@ -46,6 +46,8 @@ void AttTracer::AddPendingSignals(
     uint32_t thread_id, uint64_t queue_index
 ) {
   std::lock_guard<std::mutex> lock(sessions_pending_signals_lock_);
+  if (bIsSessionDestroying.load())
+    return;
 
   auto pending = sessions_pending_signals_.find(writer_id);
   if (pending == sessions_pending_signals_.end())
@@ -64,11 +66,18 @@ void AttTracer::AddPendingSignals(
   });
 }
 
-const std::vector<att_pending_signal_t>& AttTracer::GetPendingSignals(uint32_t writer_id) {
+std::vector<att_pending_signal_t> AttTracer::MovePendingSignals(uint32_t writer_id)
+{
   std::lock_guard<std::mutex> lock(sessions_pending_signals_lock_);
-  assert(sessions_pending_signals_.find(writer_id) != sessions_pending_signals_.end() &&
-         "writer_id is not found in the pending_signals");
-  return sessions_pending_signals_.at(writer_id);
+  auto it = sessions_pending_signals_.find(writer_id);
+  if (it == sessions_pending_signals_.end())
+    rocprofiler::fatal("writer_id is not found in the pending_signals");
+
+  auto move_pending = std::move(it->second);
+  sessions_pending_signals_.erase(writer_id);
+  if (bIsSessionDestroying.load() && sessions_pending_signals_.size() == 0)
+    has_session_pending_cv.notify_all();
+  return move_pending;
 }
 
 #define DEFAULT_ATT_BUFFER_SIZE 0x40000000
@@ -174,7 +183,8 @@ void AttTracer::signalAsyncHandlerATT(const hsa_signal_t& signal, void* data) {
     rocprofiler::fatal("Error: hsa_amd_signal_async_handler for ATT failed");
 }
 
-bool AttTracer::AsyncSignalHandlerATT(hsa_signal_value_t /* signal */, void* data) {
+bool AttTracer::AsyncSignalHandlerATT(hsa_signal_value_t /* signal */, void* data)
+{
   auto queue_info_session = static_cast<queue::queue_info_session_t*>(data);
   rocprofiler::ROCProfiler_Singleton& rocprofiler_singleton =
       rocprofiler::ROCProfiler_Singleton::GetInstance();
@@ -191,68 +201,55 @@ bool AttTracer::AsyncSignalHandlerATT(hsa_signal_value_t /* signal */, void* dat
   rocprofiler::att::AttTracer* att_tracer = session->GetAttTracer();
 
   if (!session->GetAttTracer()) return true;
+  auto pending_signals = att_tracer->MovePendingSignals(queue_info_session->writer_id);
 
-  std::vector<att_pending_signal_t>& pending_signals =
-      const_cast<std::vector<att_pending_signal_t>&>(
-          att_tracer->GetPendingSignals(queue_info_session->writer_id));
+  for (auto& pending : pending_signals)
+  {
+    rocprofiler_record_att_tracer_t record{};
+    record.kernel_id = rocprofiler_kernel_id_t{pending.kernel_descriptor};
+    record.gpu_id = rocprofiler_agent_id_t{(uint64_t)queue_info_session->agent.handle};
+    record.kernel_properties = pending.kernel_properties;
+    record.thread_id = rocprofiler_thread_id_t{pending.thread_id};
+    record.queue_idx = rocprofiler_queue_index_t{pending.queue_index};
+    record.queue_id = rocprofiler_queue_id_t{queue_info_session->queue_id};
+    record.writer_id = queue_info_session->writer_id;
 
-  if (!pending_signals.empty()) {
-    for (auto it = pending_signals.begin(); it != pending_signals.end();
-         it = pending_signals.erase(it)) {
+    if (pending.profile)
+      AddAttRecord(&record, queue_info_session->agent, pending);
 
-      auto& pending = *it;
-      //if (hsasupport_singleton.GetCoreApiTable().hsa_signal_load_relaxed_fn(pending.new_signal))
-      //  return true;
-      rocprofiler_record_att_tracer_t record{};
-      record.kernel_id = rocprofiler_kernel_id_t{pending.kernel_descriptor};
-      record.gpu_id = rocprofiler_agent_id_t{(uint64_t)queue_info_session->agent.handle};
-      record.kernel_properties = pending.kernel_properties;
-      record.thread_id = rocprofiler_thread_id_t{pending.thread_id};
-      record.queue_idx = rocprofiler_queue_index_t{pending.queue_index};
-      record.queue_id = rocprofiler_queue_id_t{queue_info_session->queue_id};
-      record.writer_id = queue_info_session->writer_id;
+    // July/01/2023 -> Changed this to queue_info_session->writer_id
+    // so we can correlate to dispatches. kernel_id already has the descriptor.
+    record.header = {ROCPROFILER_ATT_TRACER_RECORD,
+                      rocprofiler_record_id_t{pending.kernel_descriptor}};
 
-      if (/*pending.counters_count > 0 && */ pending.profile) {
-        AddAttRecord(&record, queue_info_session->agent, pending);
-      }
+    record.intercept_list = codeobj_record::get_capture(record.header.id);
+    std::atomic_thread_fence(std::memory_order_release);
 
-      // July/01/2023 -> Changed this to queue_info_session->writer_id
-      // so we can correlate to dispatches. kernel_id already has the descriptor.
-      record.header = {ROCPROFILER_ATT_TRACER_RECORD,
-                       rocprofiler_record_id_t{pending.kernel_descriptor}};
+    if (pending.session_id.handle == 0)
+      pending.session_id = rocprofiler_singleton.GetCurrentSessionId();
 
-      record.intercept_list = codeobj_record::get_capture(record.header.id);
-      std::atomic_thread_fence(std::memory_order_release);
-
-      if (pending.session_id.handle == 0) {
-        pending.session_id = rocprofiler_singleton.GetCurrentSessionId();
-      }
-
-      if (session->FindBuffer(pending.buffer_id)) {
-        Memory::GenericBuffer* buffer = session->GetBuffer(pending.buffer_id);
-        buffer->AddRecord(record);
-        buffer->Flush();
-      }
-      codeobj_record::free_capture(record.header.id);
-
-      hsa_status_t status = hsasupport_singleton.GetAmdExtTable().hsa_amd_memory_pool_free_fn(
-          (pending.profile->output_buffer.ptr));
-      if (status != HSA_STATUS_SUCCESS)
-        rocprofiler::warning("Error: Couldn't free output buffer memory");
-
-      status = hsasupport_singleton.GetAmdExtTable().hsa_amd_memory_pool_free_fn(
-          (pending.profile->command_buffer.ptr));
-      if (status != HSA_STATUS_SUCCESS)
-        rocprofiler::warning("Error: Couldn't free command buffer memory");
-
-      if (pending.profile->parameters)
-        delete[] pending.profile->parameters;
-      delete pending.profile;
+    if (session->FindBuffer(pending.buffer_id)) {
+      Memory::GenericBuffer* buffer = session->GetBuffer(pending.buffer_id);
+      buffer->AddRecord(record);
+      buffer->Flush();
     }
+    codeobj_record::free_capture(record.header.id);
+
+    hsa_status_t status = hsasupport_singleton.GetAmdExtTable().hsa_amd_memory_pool_free_fn(
+        (pending.profile->output_buffer.ptr));
+    if (status != HSA_STATUS_SUCCESS)
+      rocprofiler::warning("Error: Couldn't free output buffer memory");
+
+    status = hsasupport_singleton.GetAmdExtTable().hsa_amd_memory_pool_free_fn(
+        (pending.profile->command_buffer.ptr));
+    if (status != HSA_STATUS_SUCCESS)
+      rocprofiler::warning("Error: Couldn't free command buffer memory");
+
+    if (pending.profile->parameters)
+      delete[] pending.profile->parameters;
+    delete pending.profile;
   }
   delete queue_info_session;
-
-  std::atomic_thread_fence(std::memory_order_seq_cst);
   return false;
 }
 
@@ -320,6 +317,18 @@ hsa_status_t AttTracer::attTraceDataCallback(
   return status;
 }
 
+void AttTracer::WaitForPendingAndDestroy()
+{
+  bIsSessionDestroying.store(true);
+  std::unique_lock<std::mutex> lk(sessions_pending_signals_lock_);
+  if (sessions_pending_signals_.size() == 0)
+    return;
+
+  has_session_pending_cv.wait_for(lk, std::chrono::seconds(2), [this] () {
+    return this->sessions_pending_signals_.size() == 0;
+  });
+}
+
 std::unordered_map<uint64_t, ATTRecordSignal> AttTracer::pending_stop_packets;
 std::mutex AttTracer::att_enable_disable_mutex;
 

src/core/session/att/att.h

@@ -95,7 +95,7 @@ public:
     uint64_t agent_handle
   );
 
-  const std::vector<att_pending_signal_t>& GetPendingSignals(uint32_t writer_id);
+  std::vector<att_pending_signal_t> MovePendingSignals(uint32_t writer_id);
 
   bool ATTWriteInterceptor(
     const void* packets,
@@ -151,6 +151,8 @@ public:
     return pending_stop_packets.find(agent_handle) != pending_stop_packets.end();
   }
 
+  void WaitForPendingAndDestroy();
+
 protected:
   using packet_t = hsa_ext_amd_aql_pm4_packet_t;
   static std::unordered_map<uint64_t, ATTRecordSignal> pending_stop_packets;
@@ -208,6 +210,8 @@ private:
 
   std::mutex sessions_pending_signals_lock_;
   std::map<uint32_t, std::vector<att_pending_signal_t>> sessions_pending_signals_;
+  std::condition_variable has_session_pending_cv;
+  std::atomic<bool> bIsSessionDestroying{false};
 
   rocprofiler_record_id_t capture_id;
   std::unordered_set<uint32_t> active_capture_event_ids;

src/core/session/profiler/profiler.cpp

@@ -50,17 +50,9 @@ uint64_t GetCounterID(std::string& counter_name) {
 
 Profiler::Profiler(rocprofiler_buffer_id_t buffer_id, rocprofiler_filter_id_t filter_id,
                    rocprofiler_session_id_t session_id)
-    : buffer_id_(buffer_id), filter_id_(filter_id), session_id_(session_id) {
-  sessions_pending_signals_ = new std::map<uint32_t, std::vector<pending_signal_t*>>();
-}
-Profiler::~Profiler() {
-  for (auto& [thread_id, pending_signals] : *sessions_pending_signals_) {
-    for (auto& pending_signal : pending_signals) {
-      delete pending_signal;
-    }
-  }
-  delete sessions_pending_signals_;
-}
+    : buffer_id_(buffer_id), filter_id_(filter_id), session_id_(session_id) {}
+
+Profiler::~Profiler() {}
 
 void Profiler::AddCounterName(rocprofiler_counter_id_t counter_id, std::string counter_name) {
   std::lock_guard<std::mutex> lock(counter_names_lock_);
@@ -130,26 +122,49 @@ void Profiler::AddPendingSignals(
     rocprofiler_buffer_id_t buffer_id, rocprofiler::profiling_context_t* context,
     uint64_t session_data_count, hsa_ven_amd_aqlprofile_profile_t* profile,
     rocprofiler_kernel_properties_t kernel_properties, uint32_t thread_id, uint64_t queue_index,
-    uint64_t correlation_id) {
+    uint64_t correlation_id)
+{
   std::lock_guard<std::mutex> lock(sessions_pending_signals_lock_);
-  if (sessions_pending_signals_->find(writer_id) == sessions_pending_signals_->end())
-    sessions_pending_signals_->emplace(writer_id, std::vector<pending_signal_t*>());
-  sessions_pending_signals_->at(writer_id).emplace_back(
-      new pending_signal_t{kernel_object, original_completion_signal, new_completion_signal,
-                           session_id_, buffer_id, context, session_data_count, profile,
-                           kernel_properties, thread_id, queue_index, correlation_id});
+  if (bIsSessionDestroying.load())
+    return;
+
+  if (sessions_pending_signals_.find(writer_id) == sessions_pending_signals_.end())
+    sessions_pending_signals_.emplace(writer_id, std::vector<pending_signal_ptr_t>{});
+
+  sessions_pending_signals_.at(writer_id).emplace_back(
+    new pending_signal_t{
+      kernel_object, original_completion_signal, new_completion_signal,
+      session_id_, buffer_id, context, session_data_count, profile,
+      kernel_properties, thread_id, queue_index, correlation_id
+    }
+  );
 }
 
-const std::vector<pending_signal_t*>& Profiler::GetPendingSignals(uint32_t writer_id) {
+std::vector<pending_signal_ptr_t> Profiler::MovePendingSignals(uint32_t writer_id)
+{
   std::lock_guard<std::mutex> lock(sessions_pending_signals_lock_);
-  assert(sessions_pending_signals_->find(writer_id) != sessions_pending_signals_->end() &&
-         "writer_id is not found in the pending_signals");
-  return sessions_pending_signals_->at(writer_id);
+  auto it = sessions_pending_signals_.find(writer_id);
+  if (it == sessions_pending_signals_.end())
+    rocprofiler::fatal("writer_id is not found in the pending_signals");
+
+  auto move_pending = std::move(it->second);
+  sessions_pending_signals_.erase(writer_id);
+  if (bIsSessionDestroying.load() && sessions_pending_signals_.size() == 0)
+    has_session_pending_cv.notify_all();
+
+  return move_pending;
 }
 
-bool Profiler::CheckPendingSignalsIsEmpty() {
-  std::lock_guard<std::mutex> lock(sessions_pending_signals_lock_);
-  return sessions_pending_signals_->empty();
+void Profiler::WaitForPendingAndDestroy()
+{
+  bIsSessionDestroying.store(true);
+  std::unique_lock<std::mutex> lk(sessions_pending_signals_lock_);
+  if (sessions_pending_signals_.size() == 0)
+    return;
+
+  has_session_pending_cv.wait_for(lk, std::chrono::seconds(2), [this] () {
+    return this->sessions_pending_signals_.size() == 0;
+  });
 }
 
 }  // namespace profiler

src/core/session/profiler/profiler.h

@@ -57,6 +57,8 @@ typedef struct {
   uint64_t correlation_id;
 } pending_signal_t;
 
+typedef std::unique_ptr<pending_signal_t> pending_signal_ptr_t;
+
 namespace profiler {
 
 uint64_t GetCounterID(std::string& counter_name);
@@ -76,8 +78,8 @@ class Profiler {
                          rocprofiler_kernel_properties_t kernel_properties, uint32_t thread_id,
                          uint64_t queue_index, uint64_t correlation_id);
 
-  const std::vector<pending_signal_t*>& GetPendingSignals(uint32_t writer_id);
-  bool CheckPendingSignalsIsEmpty();
+  std::vector<pending_signal_ptr_t> MovePendingSignals(uint32_t writer_id);
+  void WaitForPendingAndDestroy();
 
   void AddCounterName(rocprofiler_counter_id_t handler, std::string counter_name);
   void AddCounterName(std::string& counter_name);
@@ -102,7 +104,9 @@ class Profiler {
   rocprofiler_session_id_t session_id_;
 
   std::mutex sessions_pending_signals_lock_;
-  std::map<uint32_t, std::vector<pending_signal_t*>>* sessions_pending_signals_;
+  std::map<uint32_t, std::vector<pending_signal_ptr_t>> sessions_pending_signals_{};
+  std::condition_variable has_session_pending_cv;
+  std::atomic<bool> bIsSessionDestroying{false};
 };
 
 }  // namespace profiler

src/core/session/session.cpp

@@ -45,52 +45,52 @@ Session::Session(rocprofiler_replay_mode_t replay_mode, rocprofiler_session_id_t
   buffers_ = new std::map<uint64_t, Memory::GenericBuffer*>();
 }
 
-Session::~Session() {
- {
+Session::~Session()
+{
     std::lock_guard<std::mutex> lock(session_lock_);
-    if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION) && spmcounter_ &&
-        spm_started_.load(std::memory_order_acquire)) {
+    if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION) && spmcounter_)
       delete spmcounter_;
-    }
-    if (FindFilterWithKind(ROCPROFILER_API_TRACE) && tracer_ &&
-        tracer_started_.load(std::memory_order_acquire)) {
+
+    if (FindFilterWithKind(ROCPROFILER_API_TRACE) && tracer_)
+    {
       delete tracer_;
-      tracer_started_.exchange(false, std::memory_order_release);
+      tracer_ = nullptr;
     }
-    if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION) && pc_sampler_ &&
-        pc_sampler_started_.load(std::memory_order_acquire)) {
+    if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION) && pc_sampler_)
+    {
       delete pc_sampler_;
-      pc_sampler_started_.exchange(false, std::memory_order_release);
+      pc_sampler_ = nullptr;
     }
 
-    if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER) && counters_sampler_ &&
-        counters_sampler_started_.load(std::memory_order_acquire)) {
+    if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER) && counters_sampler_)
+    {
       delete counters_sampler_;
-      counters_sampler_started_.exchange(false, std::memory_order_release);
+      counters_sampler_ = nullptr;
     }
     if ((FindFilterWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION) ||
-         FindFilterWithKind(ROCPROFILER_COUNTERS_COLLECTION)) &&
-        profiler_ && profiler_started_.load(std::memory_order_acquire)) {
-      rocprofiler::queue::ResetSessionID();
+         FindFilterWithKind(ROCPROFILER_COUNTERS_COLLECTION)) && profiler_)
+    {
+      rocprofiler::queue::Queue::ResetSessionID();
       delete profiler_;
-      profiler_started_.exchange(false, std::memory_order_release);
+      profiler_ = nullptr;
     }
-    if (FindFilterWithKind(ROCPROFILER_ATT_TRACE_COLLECTION) && att_tracer_ &&
-        att_tracer_started_.load(std::memory_order_acquire)) {
+    if (FindFilterWithKind(ROCPROFILER_ATT_TRACE_COLLECTION) && att_tracer_)
+    {
       delete att_tracer_;
-      att_tracer_started_.exchange(false, std::memory_order_release);
-    }
-    for (auto& filter : filters_) {
-      if (filter) delete filter;
+      att_tracer_ = nullptr;
     }
+    for (auto& filter : filters_)
+    if (filter)
+      delete filter;
+
     filters_.clear();
-    for (auto& buffer : *buffers_) {
+    for (auto& buffer : *buffers_)
+    {
       buffer.second->Flush();
       if (buffer.second) delete buffer.second;
     }
     buffers_->clear();
     if (buffers_) delete buffers_;
-  }
 }
 
 void Session::DisableTools(rocprofiler_buffer_id_t buffer_id) {
@@ -104,134 +104,141 @@ void Session::DisableTools(rocprofiler_buffer_id_t buffer_id) {
   }
   if (FindFilterWithKind(ROCPROFILER_API_TRACE) &&
       GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetBufferId().value ==
-          buffer_id.value) {
-    if (tracer_started_.load(std::memory_order_acquire)) {
+          buffer_id.value)
+  {
+    if (tracer_)
       tracer_->DisableRoctracer();
-    }
   }
 }
 
-void Session::Start() {
+void Session::Start()
+{
   std::lock_guard<std::mutex> lock(session_lock_);
-  if (!is_active_) {
-    if (!profiler_started_.load(std::memory_order_acquire)) {
-      if (FindFilterWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION)) {
-        profiler_ = new profiler::Profiler(
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION))
-                ->GetBufferId(),
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION))->GetId(),
-            session_id_);
-        profiler_started_.exchange(true, std::memory_order_release);
-      }
+  if (is_active_) return;
 
-      if (FindFilterWithKind(ROCPROFILER_COUNTERS_COLLECTION)) {
-        profiler_ = new profiler::Profiler(
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_COLLECTION))->GetBufferId(),
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_COLLECTION))->GetId(), session_id_);
-        profiler_started_.exchange(true, std::memory_order_release);
-      }
-    } else {
-      rocprofiler::queue::ResetSessionID(session_id_);
+  if (!profiler_)
+  {
+    if (FindFilterWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION))
+    {
+      profiler_ = new profiler::Profiler(
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION))
+              ->GetBufferId(),
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION))->GetId(),
+          session_id_);
     }
-    if (FindFilterWithKind(ROCPROFILER_ATT_TRACE_COLLECTION)) {
-      if (!att_tracer_started_.load(std::memory_order_acquire)) {
-        att_tracer_ = new att::AttTracer(
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_ATT_TRACE_COLLECTION))->GetBufferId(),
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_ATT_TRACE_COLLECTION))->GetId(), session_id_);
-        att_tracer_started_.exchange(true, std::memory_order_release);
-      }
+    if (FindFilterWithKind(ROCPROFILER_COUNTERS_COLLECTION))
+    {
+      profiler_ = new profiler::Profiler(
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_COLLECTION))->GetBufferId(),
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_COLLECTION))->GetId(), session_id_);
     }
-
-    if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION)) {
-      if (!spm_started_.load(std::memory_order_acquire)) {
-        rocprofiler_spm_parameter_t* spmparameter =
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetSpmParameterData();
-        spmcounter_ = new spm::SpmCounters(
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetBufferId(),
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetId(), spmparameter,
-            session_id_);
-      }
-      if (!profiler_started_.load(std::memory_order_acquire)) {
-        profiler_ = new profiler::Profiler(
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetBufferId(),
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetId(), session_id_);
-        profiler_started_.exchange(true, std::memory_order_release);
-      }
-      startSpm();
-    }
-
-    if (FindFilterWithKind(ROCPROFILER_API_TRACE)) {
-      std::vector<rocprofiler_tracer_activity_domain_t> domains =
-          GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetTraceData();
-      if (!tracer_started_.load(std::memory_order_acquire)) {
-        tracer_ = new tracer::Tracer(
-            session_id_,
-            (GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->HasCallback()
-                 ? GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetCallback()
-                 : nullptr),
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetBufferId(), domains);
-        tracer_started_.exchange(true, std::memory_order_release);
-      }
-      tracer_->StartRoctracer();
-    }
-
-    if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION)) {
-      if (!pc_sampler_started_.load(std::memory_order_acquire)) {
-        pc_sampler_ = new pc_sampler::PCSampler(
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION))->GetBufferId(),
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION))->GetId(),
-            session_id_);
-        pc_sampler_started_.exchange(true, std::memory_order_release);
-      }
-      pc_sampler_->Start();
-    }
-
-    if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER)) {
-      if (!counters_sampler_started_.load(std::memory_order_acquire)) {
-        counters_sampler_ = new CountersSampler(
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_SAMPLER))->GetBufferId(),
-            GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_SAMPLER))->GetId(), session_id_);
-        counters_sampler_started_.exchange(true, std::memory_order_release);
-      }
-      counters_sampler_->Start();
-    }
-
-    is_active_ = true;
   }
+  else
+  {
+    rocprofiler::queue::Queue::ResetSessionID(session_id_);
+  }
+
+  if (FindFilterWithKind(ROCPROFILER_ATT_TRACE_COLLECTION) && !att_tracer_)
+  {
+    att_tracer_ = new att::AttTracer(
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_ATT_TRACE_COLLECTION))->GetBufferId(),
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_ATT_TRACE_COLLECTION))->GetId(), session_id_);
+  }
+
+  if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION)) {
+    if (!spm_started_)
+    {
+      rocprofiler_spm_parameter_t* spmparameter =
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetSpmParameterData();
+      spmcounter_ = new spm::SpmCounters(
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetBufferId(),
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetId(), spmparameter,
+          session_id_);
+    }
+    if (!profiler_)
+    {
+      profiler_ = new profiler::Profiler(
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetBufferId(),
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetId(), session_id_);
+    }
+    startSpm();
+  }
+
+  if (FindFilterWithKind(ROCPROFILER_API_TRACE))
+  {
+    std::vector<rocprofiler_tracer_activity_domain_t> domains =
+        GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetTraceData();
+    if (!tracer_)
+    {
+      tracer_ = new tracer::Tracer(
+          session_id_,
+          (GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->HasCallback()
+                ? GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetCallback()
+                : nullptr),
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetBufferId(), domains);
+    }
+    tracer_->StartRoctracer();
+  }
+
+  if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION))
+  {
+    if (!pc_sampler_)
+    {
+      pc_sampler_ = new pc_sampler::PCSampler(
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION))->GetBufferId(),
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION))->GetId(),
+          session_id_);
+    }
+    pc_sampler_->Start();
+  }
+
+  if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER))
+  {
+    if (!counters_sampler_)
+    {
+      counters_sampler_ = new CountersSampler(
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_SAMPLER))->GetBufferId(),
+          GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_SAMPLER))->GetId(), session_id_);
+    }
+    counters_sampler_->Start();
+  }
+
+  is_active_ = true;
 }
 
-void Session::Terminate() {
-  if (is_active_) {
-    rocprofiler::queue::ResetSessionID();
-    std::lock_guard<std::mutex> lock(session_lock_);
-    if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION)) {
-      { stopSpm(); }
-    }
-    if (FindFilterWithKind(ROCPROFILER_API_TRACE)) {
-      std::vector<rocprofiler_tracer_activity_domain_t> domains =
-          GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetTraceData();
-      if (tracer_started_.load(std::memory_order_acquire)) {
-        tracer_->StopRoctracer();
-      }
-    }
-    if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION)) {
-      if (pc_sampler_started_.load(std::memory_order_acquire)) {
-        pc_sampler_->Stop();
-      }
-    }
+void Session::Terminate()
+{
+  if (!is_active_.load(std::memory_order_acquire))
+    return;
 
-    if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER)) {
-      if (counters_sampler_started_.load(std::memory_order_acquire)) {
-        counters_sampler_->Stop();
-      }
-    }
+  rocprofiler::queue::Queue::ResetSessionID();
+  if (GetProfiler())
+    GetProfiler()->WaitForPendingAndDestroy();
+  if (GetAttTracer())
+    GetAttTracer()->WaitForPendingAndDestroy();
 
-    for (auto& buffer : *buffers_) {
-      buffer.second->Flush();
-    }
 
-    is_active_ = false;
+  std::lock_guard<std::mutex> lock(session_lock_);
+  if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION)) {
+    { stopSpm(); }
   }
+  if (FindFilterWithKind(ROCPROFILER_API_TRACE))
+  {
+    std::vector<rocprofiler_tracer_activity_domain_t> domains =
+        GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetTraceData();
+    if (tracer_)
+      tracer_->StopRoctracer();
+  }
+  if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION) && pc_sampler_)
+    pc_sampler_->Stop();
+
+  if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER) && counters_sampler_)
+    counters_sampler_->Stop();
+
+  for (auto& buffer : *buffers_)
+    buffer.second->Flush();
+
+  is_active_.store(false, std::memory_order_relaxed);
 }
 
 rocprofiler_session_id_t Session::GetId() { return session_id_; }
@@ -352,7 +359,7 @@ void Session::DestroyBuffer(rocprofiler_buffer_id_t buffer_id) {
 
 rocprofiler_status_t Session::startSpm() {
   if (spmcounter_) {
-    spm_started_.exchange(true, std::memory_order_release);
+    spm_started_ = true;
     return spmcounter_->startSpm();
   } else {
     std::cout << "Apply the SPM Filter" << std::endl;
@@ -361,8 +368,8 @@ rocprofiler_status_t Session::startSpm() {
 }
 
 rocprofiler_status_t Session::stopSpm() {
-  if (spmcounter_ && spm_started_.load()) {
-    spm_started_.exchange(false, std::memory_order_release);
+  if (spmcounter_ && spm_started_) {
+    spm_started_ = false;
     return spmcounter_->stopSpm();
   } else {
     std::cout << "SPM not started" << std::endl;

src/core/session/session.h

@@ -106,25 +106,19 @@ class Session {
   std::mutex filters_lock_;
   std::vector<Filter*> filters_;
 
-  std::atomic<bool> profiler_started_{false};
-  std::atomic<bool> tracer_started_{false};
-  std::atomic<bool> att_tracer_started_{false};
-  att::AttTracer* att_tracer_;
-  std::atomic<bool> spm_started_{false};
+  att::AttTracer* att_tracer_ = nullptr;
+  bool spm_started_{false};
 
-  profiler::Profiler* profiler_;
-  tracer::Tracer* tracer_;
-  spm::SpmCounters* spmcounter_;
+  profiler::Profiler* profiler_ = nullptr;
+  tracer::Tracer* tracer_ = nullptr;
+  spm::SpmCounters* spmcounter_ = nullptr;
 
-  std::atomic<bool> pc_sampler_started_{false};
-  pc_sampler::PCSampler* pc_sampler_;
-
-  std::atomic<bool> counters_sampler_started_{false};
-  CountersSampler* counters_sampler_;
+  pc_sampler::PCSampler* pc_sampler_ = nullptr;
+  CountersSampler* counters_sampler_ = nullptr;
 
   std::atomic<uint64_t> buffers_counter_{1};
   std::mutex buffers_lock_;
-  std::map<uint64_t, Memory::GenericBuffer*>* buffers_;
+  std::map<uint64_t, Memory::GenericBuffer*>* buffers_ = nullptr;
   std::atomic<uint64_t> records_counter_{1};