diff --git a/src/core/hsa/queues/queue.cpp b/src/core/hsa/queues/queue.cpp index a247669840..1b43260953 100644 --- a/src/core/hsa/queues/queue.cpp +++ b/src/core/hsa/queues/queue.cpp @@ -384,25 +384,27 @@ void SignalAsyncReadyHandler(const hsa_signal_t& signal, void* data) { signal, HSA_SIGNAL_CONDITION_EQ, 0, AsyncSignalReadyHandler, data); if (status != HSA_STATUS_SUCCESS) fatal("hsa_amd_signal_async_handler failed"); } -bool AsyncSignalHandler(hsa_signal_value_t signal_value, void* data) { +bool AsyncSignalHandler(hsa_signal_value_t signal_value, void* data) +{ auto queue_info_session = static_cast(data); + if (!queue_info_session) return true; + rocprofiler::ROCProfiler_Singleton& rocprofiler_singleton = - rocprofiler::ROCProfiler_Singleton::GetInstance(); + rocprofiler::ROCProfiler_Singleton::GetInstance(); rocprofiler::HSASupport_Singleton& hsasupport_singleton = - rocprofiler::HSASupport_Singleton::GetInstance(); - if (!queue_info_session || !rocprofiler_singleton.GetSession(queue_info_session->session_id) || - !rocprofiler_singleton.GetSession(queue_info_session->session_id)->GetProfiler()) - return true; + rocprofiler::HSASupport_Singleton::GetInstance(); + rocprofiler::Session* session = rocprofiler_singleton.GetSession(queue_info_session->session_id); + if (!session) return true; + std::lock_guard lock(session->GetSessionLock()); rocprofiler::profiler::Profiler* profiler = session->GetProfiler(); - std::vector pending_signals = const_cast&>( - profiler->GetPendingSignals(queue_info_session->writer_id)); + if (!profiler) return true; - if (!pending_signals.empty()) { - for (auto it = pending_signals.begin(); it != pending_signals.end(); - it = pending_signals.erase(it)) { - auto& pending = *it; + auto pending_signals = profiler->MovePendingSignals(queue_info_session->writer_id); + + for (auto& pending : pending_signals) + { if (hsasupport_singleton.GetCoreApiTable().hsa_signal_load_relaxed_fn(pending->new_signal)) return true; hsa_amd_profiling_dispatch_time_t time; @@ -458,7 +460,7 @@ bool AsyncSignalHandler(hsa_signal_value_t signal_value, void* data) { rocprofiler::metrics::GetMetricsData(pending->context->results_map, pending->context->metrics_list, time.end - time.start); - AddRecordCounters(&record, pending); + AddRecordCounters(&record, pending.get()); } else { if (session->FindBuffer(pending->buffer_id)) { Memory::GenericBuffer* buffer = session->GetBuffer(pending->buffer_id); @@ -503,14 +505,11 @@ bool AsyncSignalHandler(hsa_signal_value_t signal_value, void* data) { } - - if (pending->new_signal.handle) hsasupport_singleton.GetCoreApiTable().hsa_signal_destroy_fn(pending->new_signal); if (queue_info_session->interrupt_signal.handle) hsasupport_singleton.GetCoreApiTable().hsa_signal_destroy_fn( queue_info_session->interrupt_signal); - } } delete queue_info_session; ACTIVE_INTERRUPT_SIGNAL_COUNT.fetch_sub(1, std::memory_order_relaxed); @@ -529,7 +528,8 @@ void CreateSignal(uint32_t attribute, hsa_signal_t* signal) { HSASupport_Singleton::GetInstance().CreateSignal(attribute, signal); } -rocprofiler_session_id_t session_id = rocprofiler_session_id_t{0}; +rocprofiler_session_id_t Queue::session_id = rocprofiler_session_id_t{0}; +std::shared_mutex Queue::session_id_mutex; // Counter Names declaration std::vector session_data; @@ -546,9 +546,13 @@ uint32_t replay_mode_count = 0; rocprofiler::Session* session = nullptr; -void ResetSessionID(rocprofiler_session_id_t id) { session_id = id; } +void Queue::ResetSessionID(rocprofiler_session_id_t id) +{ + std::unique_lock session_id_lock(session_id_mutex); + session_id = id; +} -void CheckNeededProfileConfigs() { +void Queue::CheckNeededProfileConfigs() { rocprofiler_session_id_t internal_session_id; // Getting Session ID rocprofiler::ROCProfiler_Singleton& rocprofiler_singleton = @@ -609,7 +613,9 @@ std::atomic WRITER_ID{0}; * interceptor by invoking the writer function. */ void Queue::WriteInterceptor(const void* packets, uint64_t pkt_count, uint64_t user_pkt_index, - void* data, hsa_amd_queue_intercept_packet_writer writer) { + void* data, hsa_amd_queue_intercept_packet_writer writer) +{ + std::shared_lock session_id_lock(session_id_mutex); const Packet::packet_t* packets_arr = reinterpret_cast(packets); std::vector transformed_packets; @@ -669,7 +675,6 @@ void Queue::WriteInterceptor(const void* packets, uint64_t pkt_count, uint64_t u Packet::CreateBarrierPacket(&transformed_packets, &block_signal, &block_signal); } - uint32_t writer_id = WRITER_ID.fetch_add(1, std::memory_order_release); if (session_data_count > 0 && is_counter_collection_mode && profiles.size() > 0 && diff --git a/src/core/hsa/queues/queue.h b/src/core/hsa/queues/queue.h index d49fa6c80e..f2dbb7fa55 100644 --- a/src/core/hsa/queues/queue.h +++ b/src/core/hsa/queues/queue.h @@ -35,6 +35,7 @@ #include #include #include +#include #include "src/core/session/profiler/profiler.h" namespace rocprofiler { @@ -87,7 +88,11 @@ class Queue { hsa_signal_t GetReadySignal(); hsa_signal_t GetBlockSignal(); + static void ResetSessionID(rocprofiler_session_id_t id = rocprofiler_session_id_t{0}); + static void CheckNeededProfileConfigs(); private: + static std::shared_mutex session_id_mutex; + static rocprofiler_session_id_t session_id; hsa_agent_t cpu_agent_; hsa_agent_t gpu_agent_; @@ -113,8 +118,6 @@ struct queue_info_session_t { void AddRecordCounters(rocprofiler_record_profiler_t* record, const pending_signal_t& pending); -void ResetSessionID(rocprofiler_session_id_t id = rocprofiler_session_id_t{0}); - void CheckPacketReqiurements(); } // namespace queue diff --git a/src/core/session/att/att.cpp b/src/core/session/att/att.cpp index ffca5bd660..48368f13d0 100644 --- a/src/core/session/att/att.cpp +++ b/src/core/session/att/att.cpp @@ -46,6 +46,8 @@ void AttTracer::AddPendingSignals( uint32_t thread_id, uint64_t queue_index ) { std::lock_guard lock(sessions_pending_signals_lock_); + if (bIsSessionDestroying.load()) + return; auto pending = sessions_pending_signals_.find(writer_id); if (pending == sessions_pending_signals_.end()) @@ -64,11 +66,18 @@ void AttTracer::AddPendingSignals( }); } -const std::vector& AttTracer::GetPendingSignals(uint32_t writer_id) { +std::vector AttTracer::MovePendingSignals(uint32_t writer_id) +{ std::lock_guard lock(sessions_pending_signals_lock_); - assert(sessions_pending_signals_.find(writer_id) != sessions_pending_signals_.end() && - "writer_id is not found in the pending_signals"); - return sessions_pending_signals_.at(writer_id); + auto it = sessions_pending_signals_.find(writer_id); + if (it == sessions_pending_signals_.end()) + rocprofiler::fatal("writer_id is not found in the pending_signals"); + + auto move_pending = std::move(it->second); + sessions_pending_signals_.erase(writer_id); + if (bIsSessionDestroying.load() && sessions_pending_signals_.size() == 0) + has_session_pending_cv.notify_all(); + return move_pending; } #define DEFAULT_ATT_BUFFER_SIZE 0x40000000 @@ -174,7 +183,8 @@ void AttTracer::signalAsyncHandlerATT(const hsa_signal_t& signal, void* data) { rocprofiler::fatal("Error: hsa_amd_signal_async_handler for ATT failed"); } -bool AttTracer::AsyncSignalHandlerATT(hsa_signal_value_t /* signal */, void* data) { +bool AttTracer::AsyncSignalHandlerATT(hsa_signal_value_t /* signal */, void* data) +{ auto queue_info_session = static_cast(data); rocprofiler::ROCProfiler_Singleton& rocprofiler_singleton = rocprofiler::ROCProfiler_Singleton::GetInstance(); @@ -191,68 +201,55 @@ bool AttTracer::AsyncSignalHandlerATT(hsa_signal_value_t /* signal */, void* dat rocprofiler::att::AttTracer* att_tracer = session->GetAttTracer(); if (!session->GetAttTracer()) return true; + auto pending_signals = att_tracer->MovePendingSignals(queue_info_session->writer_id); - std::vector& pending_signals = - const_cast&>( - att_tracer->GetPendingSignals(queue_info_session->writer_id)); + for (auto& pending : pending_signals) + { + rocprofiler_record_att_tracer_t record{}; + record.kernel_id = rocprofiler_kernel_id_t{pending.kernel_descriptor}; + record.gpu_id = rocprofiler_agent_id_t{(uint64_t)queue_info_session->agent.handle}; + record.kernel_properties = pending.kernel_properties; + record.thread_id = rocprofiler_thread_id_t{pending.thread_id}; + record.queue_idx = rocprofiler_queue_index_t{pending.queue_index}; + record.queue_id = rocprofiler_queue_id_t{queue_info_session->queue_id}; + record.writer_id = queue_info_session->writer_id; - if (!pending_signals.empty()) { - for (auto it = pending_signals.begin(); it != pending_signals.end(); - it = pending_signals.erase(it)) { + if (pending.profile) + AddAttRecord(&record, queue_info_session->agent, pending); - auto& pending = *it; - //if (hsasupport_singleton.GetCoreApiTable().hsa_signal_load_relaxed_fn(pending.new_signal)) - // return true; - rocprofiler_record_att_tracer_t record{}; - record.kernel_id = rocprofiler_kernel_id_t{pending.kernel_descriptor}; - record.gpu_id = rocprofiler_agent_id_t{(uint64_t)queue_info_session->agent.handle}; - record.kernel_properties = pending.kernel_properties; - record.thread_id = rocprofiler_thread_id_t{pending.thread_id}; - record.queue_idx = rocprofiler_queue_index_t{pending.queue_index}; - record.queue_id = rocprofiler_queue_id_t{queue_info_session->queue_id}; - record.writer_id = queue_info_session->writer_id; + // July/01/2023 -> Changed this to queue_info_session->writer_id + // so we can correlate to dispatches. kernel_id already has the descriptor. + record.header = {ROCPROFILER_ATT_TRACER_RECORD, + rocprofiler_record_id_t{pending.kernel_descriptor}}; - if (/*pending.counters_count > 0 && */ pending.profile) { - AddAttRecord(&record, queue_info_session->agent, pending); - } + record.intercept_list = codeobj_record::get_capture(record.header.id); + std::atomic_thread_fence(std::memory_order_release); - // July/01/2023 -> Changed this to queue_info_session->writer_id - // so we can correlate to dispatches. kernel_id already has the descriptor. - record.header = {ROCPROFILER_ATT_TRACER_RECORD, - rocprofiler_record_id_t{pending.kernel_descriptor}}; + if (pending.session_id.handle == 0) + pending.session_id = rocprofiler_singleton.GetCurrentSessionId(); - record.intercept_list = codeobj_record::get_capture(record.header.id); - std::atomic_thread_fence(std::memory_order_release); - - if (pending.session_id.handle == 0) { - pending.session_id = rocprofiler_singleton.GetCurrentSessionId(); - } - - if (session->FindBuffer(pending.buffer_id)) { - Memory::GenericBuffer* buffer = session->GetBuffer(pending.buffer_id); - buffer->AddRecord(record); - buffer->Flush(); - } - codeobj_record::free_capture(record.header.id); - - hsa_status_t status = hsasupport_singleton.GetAmdExtTable().hsa_amd_memory_pool_free_fn( - (pending.profile->output_buffer.ptr)); - if (status != HSA_STATUS_SUCCESS) - rocprofiler::warning("Error: Couldn't free output buffer memory"); - - status = hsasupport_singleton.GetAmdExtTable().hsa_amd_memory_pool_free_fn( - (pending.profile->command_buffer.ptr)); - if (status != HSA_STATUS_SUCCESS) - rocprofiler::warning("Error: Couldn't free command buffer memory"); - - if (pending.profile->parameters) - delete[] pending.profile->parameters; - delete pending.profile; + if (session->FindBuffer(pending.buffer_id)) { + Memory::GenericBuffer* buffer = session->GetBuffer(pending.buffer_id); + buffer->AddRecord(record); + buffer->Flush(); } + codeobj_record::free_capture(record.header.id); + + hsa_status_t status = hsasupport_singleton.GetAmdExtTable().hsa_amd_memory_pool_free_fn( + (pending.profile->output_buffer.ptr)); + if (status != HSA_STATUS_SUCCESS) + rocprofiler::warning("Error: Couldn't free output buffer memory"); + + status = hsasupport_singleton.GetAmdExtTable().hsa_amd_memory_pool_free_fn( + (pending.profile->command_buffer.ptr)); + if (status != HSA_STATUS_SUCCESS) + rocprofiler::warning("Error: Couldn't free command buffer memory"); + + if (pending.profile->parameters) + delete[] pending.profile->parameters; + delete pending.profile; } delete queue_info_session; - - std::atomic_thread_fence(std::memory_order_seq_cst); return false; } @@ -320,6 +317,18 @@ hsa_status_t AttTracer::attTraceDataCallback( return status; } +void AttTracer::WaitForPendingAndDestroy() +{ + bIsSessionDestroying.store(true); + std::unique_lock lk(sessions_pending_signals_lock_); + if (sessions_pending_signals_.size() == 0) + return; + + has_session_pending_cv.wait_for(lk, std::chrono::seconds(2), [this] () { + return this->sessions_pending_signals_.size() == 0; + }); +} + std::unordered_map AttTracer::pending_stop_packets; std::mutex AttTracer::att_enable_disable_mutex; diff --git a/src/core/session/att/att.h b/src/core/session/att/att.h index 523b287378..428f85f147 100644 --- a/src/core/session/att/att.h +++ b/src/core/session/att/att.h @@ -95,7 +95,7 @@ public: uint64_t agent_handle ); - const std::vector& GetPendingSignals(uint32_t writer_id); + std::vector MovePendingSignals(uint32_t writer_id); bool ATTWriteInterceptor( const void* packets, @@ -151,6 +151,8 @@ public: return pending_stop_packets.find(agent_handle) != pending_stop_packets.end(); } + void WaitForPendingAndDestroy(); + protected: using packet_t = hsa_ext_amd_aql_pm4_packet_t; static std::unordered_map pending_stop_packets; @@ -208,6 +210,8 @@ private: std::mutex sessions_pending_signals_lock_; std::map> sessions_pending_signals_; + std::condition_variable has_session_pending_cv; + std::atomic bIsSessionDestroying{false}; rocprofiler_record_id_t capture_id; std::unordered_set active_capture_event_ids; diff --git a/src/core/session/profiler/profiler.cpp b/src/core/session/profiler/profiler.cpp index 32d4186a50..2fa0f838c5 100644 --- a/src/core/session/profiler/profiler.cpp +++ b/src/core/session/profiler/profiler.cpp @@ -50,17 +50,9 @@ uint64_t GetCounterID(std::string& counter_name) { Profiler::Profiler(rocprofiler_buffer_id_t buffer_id, rocprofiler_filter_id_t filter_id, rocprofiler_session_id_t session_id) - : buffer_id_(buffer_id), filter_id_(filter_id), session_id_(session_id) { - sessions_pending_signals_ = new std::map>(); -} -Profiler::~Profiler() { - for (auto& [thread_id, pending_signals] : *sessions_pending_signals_) { - for (auto& pending_signal : pending_signals) { - delete pending_signal; - } - } - delete sessions_pending_signals_; -} + : buffer_id_(buffer_id), filter_id_(filter_id), session_id_(session_id) {} + +Profiler::~Profiler() {} void Profiler::AddCounterName(rocprofiler_counter_id_t counter_id, std::string counter_name) { std::lock_guard lock(counter_names_lock_); @@ -130,26 +122,49 @@ void Profiler::AddPendingSignals( rocprofiler_buffer_id_t buffer_id, rocprofiler::profiling_context_t* context, uint64_t session_data_count, hsa_ven_amd_aqlprofile_profile_t* profile, rocprofiler_kernel_properties_t kernel_properties, uint32_t thread_id, uint64_t queue_index, - uint64_t correlation_id) { + uint64_t correlation_id) +{ std::lock_guard lock(sessions_pending_signals_lock_); - if (sessions_pending_signals_->find(writer_id) == sessions_pending_signals_->end()) - sessions_pending_signals_->emplace(writer_id, std::vector()); - sessions_pending_signals_->at(writer_id).emplace_back( - new pending_signal_t{kernel_object, original_completion_signal, new_completion_signal, - session_id_, buffer_id, context, session_data_count, profile, - kernel_properties, thread_id, queue_index, correlation_id}); + if (bIsSessionDestroying.load()) + return; + + if (sessions_pending_signals_.find(writer_id) == sessions_pending_signals_.end()) + sessions_pending_signals_.emplace(writer_id, std::vector{}); + + sessions_pending_signals_.at(writer_id).emplace_back( + new pending_signal_t{ + kernel_object, original_completion_signal, new_completion_signal, + session_id_, buffer_id, context, session_data_count, profile, + kernel_properties, thread_id, queue_index, correlation_id + } + ); } -const std::vector& Profiler::GetPendingSignals(uint32_t writer_id) { +std::vector Profiler::MovePendingSignals(uint32_t writer_id) +{ std::lock_guard lock(sessions_pending_signals_lock_); - assert(sessions_pending_signals_->find(writer_id) != sessions_pending_signals_->end() && - "writer_id is not found in the pending_signals"); - return sessions_pending_signals_->at(writer_id); + auto it = sessions_pending_signals_.find(writer_id); + if (it == sessions_pending_signals_.end()) + rocprofiler::fatal("writer_id is not found in the pending_signals"); + + auto move_pending = std::move(it->second); + sessions_pending_signals_.erase(writer_id); + if (bIsSessionDestroying.load() && sessions_pending_signals_.size() == 0) + has_session_pending_cv.notify_all(); + + return move_pending; } -bool Profiler::CheckPendingSignalsIsEmpty() { - std::lock_guard lock(sessions_pending_signals_lock_); - return sessions_pending_signals_->empty(); +void Profiler::WaitForPendingAndDestroy() +{ + bIsSessionDestroying.store(true); + std::unique_lock lk(sessions_pending_signals_lock_); + if (sessions_pending_signals_.size() == 0) + return; + + has_session_pending_cv.wait_for(lk, std::chrono::seconds(2), [this] () { + return this->sessions_pending_signals_.size() == 0; + }); } } // namespace profiler diff --git a/src/core/session/profiler/profiler.h b/src/core/session/profiler/profiler.h index 5b9dca8b87..aa5c389296 100644 --- a/src/core/session/profiler/profiler.h +++ b/src/core/session/profiler/profiler.h @@ -57,6 +57,8 @@ typedef struct { uint64_t correlation_id; } pending_signal_t; +typedef std::unique_ptr pending_signal_ptr_t; + namespace profiler { uint64_t GetCounterID(std::string& counter_name); @@ -76,8 +78,8 @@ class Profiler { rocprofiler_kernel_properties_t kernel_properties, uint32_t thread_id, uint64_t queue_index, uint64_t correlation_id); - const std::vector& GetPendingSignals(uint32_t writer_id); - bool CheckPendingSignalsIsEmpty(); + std::vector MovePendingSignals(uint32_t writer_id); + void WaitForPendingAndDestroy(); void AddCounterName(rocprofiler_counter_id_t handler, std::string counter_name); void AddCounterName(std::string& counter_name); @@ -102,7 +104,9 @@ class Profiler { rocprofiler_session_id_t session_id_; std::mutex sessions_pending_signals_lock_; - std::map>* sessions_pending_signals_; + std::map> sessions_pending_signals_{}; + std::condition_variable has_session_pending_cv; + std::atomic bIsSessionDestroying{false}; }; } // namespace profiler diff --git a/src/core/session/session.cpp b/src/core/session/session.cpp index 39428e6ac3..4fea3b4489 100644 --- a/src/core/session/session.cpp +++ b/src/core/session/session.cpp @@ -45,52 +45,52 @@ Session::Session(rocprofiler_replay_mode_t replay_mode, rocprofiler_session_id_t buffers_ = new std::map(); } -Session::~Session() { - { +Session::~Session() +{ std::lock_guard lock(session_lock_); - if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION) && spmcounter_ && - spm_started_.load(std::memory_order_acquire)) { + if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION) && spmcounter_) delete spmcounter_; - } - if (FindFilterWithKind(ROCPROFILER_API_TRACE) && tracer_ && - tracer_started_.load(std::memory_order_acquire)) { + + if (FindFilterWithKind(ROCPROFILER_API_TRACE) && tracer_) + { delete tracer_; - tracer_started_.exchange(false, std::memory_order_release); + tracer_ = nullptr; } - if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION) && pc_sampler_ && - pc_sampler_started_.load(std::memory_order_acquire)) { + if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION) && pc_sampler_) + { delete pc_sampler_; - pc_sampler_started_.exchange(false, std::memory_order_release); + pc_sampler_ = nullptr; } - if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER) && counters_sampler_ && - counters_sampler_started_.load(std::memory_order_acquire)) { + if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER) && counters_sampler_) + { delete counters_sampler_; - counters_sampler_started_.exchange(false, std::memory_order_release); + counters_sampler_ = nullptr; } if ((FindFilterWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION) || - FindFilterWithKind(ROCPROFILER_COUNTERS_COLLECTION)) && - profiler_ && profiler_started_.load(std::memory_order_acquire)) { - rocprofiler::queue::ResetSessionID(); + FindFilterWithKind(ROCPROFILER_COUNTERS_COLLECTION)) && profiler_) + { + rocprofiler::queue::Queue::ResetSessionID(); delete profiler_; - profiler_started_.exchange(false, std::memory_order_release); + profiler_ = nullptr; } - if (FindFilterWithKind(ROCPROFILER_ATT_TRACE_COLLECTION) && att_tracer_ && - att_tracer_started_.load(std::memory_order_acquire)) { + if (FindFilterWithKind(ROCPROFILER_ATT_TRACE_COLLECTION) && att_tracer_) + { delete att_tracer_; - att_tracer_started_.exchange(false, std::memory_order_release); - } - for (auto& filter : filters_) { - if (filter) delete filter; + att_tracer_ = nullptr; } + for (auto& filter : filters_) + if (filter) + delete filter; + filters_.clear(); - for (auto& buffer : *buffers_) { + for (auto& buffer : *buffers_) + { buffer.second->Flush(); if (buffer.second) delete buffer.second; } buffers_->clear(); if (buffers_) delete buffers_; - } } void Session::DisableTools(rocprofiler_buffer_id_t buffer_id) { @@ -104,134 +104,141 @@ void Session::DisableTools(rocprofiler_buffer_id_t buffer_id) { } if (FindFilterWithKind(ROCPROFILER_API_TRACE) && GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetBufferId().value == - buffer_id.value) { - if (tracer_started_.load(std::memory_order_acquire)) { + buffer_id.value) + { + if (tracer_) tracer_->DisableRoctracer(); - } } } -void Session::Start() { +void Session::Start() +{ std::lock_guard lock(session_lock_); - if (!is_active_) { - if (!profiler_started_.load(std::memory_order_acquire)) { - if (FindFilterWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION)) { - profiler_ = new profiler::Profiler( - GetFilter(GetFilterIdWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION)) - ->GetBufferId(), - GetFilter(GetFilterIdWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION))->GetId(), - session_id_); - profiler_started_.exchange(true, std::memory_order_release); - } + if (is_active_) return; - if (FindFilterWithKind(ROCPROFILER_COUNTERS_COLLECTION)) { - profiler_ = new profiler::Profiler( - GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_COLLECTION))->GetBufferId(), - GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_COLLECTION))->GetId(), session_id_); - profiler_started_.exchange(true, std::memory_order_release); - } - } else { - rocprofiler::queue::ResetSessionID(session_id_); + if (!profiler_) + { + if (FindFilterWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION)) + { + profiler_ = new profiler::Profiler( + GetFilter(GetFilterIdWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION)) + ->GetBufferId(), + GetFilter(GetFilterIdWithKind(ROCPROFILER_DISPATCH_TIMESTAMPS_COLLECTION))->GetId(), + session_id_); } - if (FindFilterWithKind(ROCPROFILER_ATT_TRACE_COLLECTION)) { - if (!att_tracer_started_.load(std::memory_order_acquire)) { - att_tracer_ = new att::AttTracer( - GetFilter(GetFilterIdWithKind(ROCPROFILER_ATT_TRACE_COLLECTION))->GetBufferId(), - GetFilter(GetFilterIdWithKind(ROCPROFILER_ATT_TRACE_COLLECTION))->GetId(), session_id_); - att_tracer_started_.exchange(true, std::memory_order_release); - } + if (FindFilterWithKind(ROCPROFILER_COUNTERS_COLLECTION)) + { + profiler_ = new profiler::Profiler( + GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_COLLECTION))->GetBufferId(), + GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_COLLECTION))->GetId(), session_id_); } - - if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION)) { - if (!spm_started_.load(std::memory_order_acquire)) { - rocprofiler_spm_parameter_t* spmparameter = - GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetSpmParameterData(); - spmcounter_ = new spm::SpmCounters( - GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetBufferId(), - GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetId(), spmparameter, - session_id_); - } - if (!profiler_started_.load(std::memory_order_acquire)) { - profiler_ = new profiler::Profiler( - GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetBufferId(), - GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetId(), session_id_); - profiler_started_.exchange(true, std::memory_order_release); - } - startSpm(); - } - - if (FindFilterWithKind(ROCPROFILER_API_TRACE)) { - std::vector domains = - GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetTraceData(); - if (!tracer_started_.load(std::memory_order_acquire)) { - tracer_ = new tracer::Tracer( - session_id_, - (GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->HasCallback() - ? GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetCallback() - : nullptr), - GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetBufferId(), domains); - tracer_started_.exchange(true, std::memory_order_release); - } - tracer_->StartRoctracer(); - } - - if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION)) { - if (!pc_sampler_started_.load(std::memory_order_acquire)) { - pc_sampler_ = new pc_sampler::PCSampler( - GetFilter(GetFilterIdWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION))->GetBufferId(), - GetFilter(GetFilterIdWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION))->GetId(), - session_id_); - pc_sampler_started_.exchange(true, std::memory_order_release); - } - pc_sampler_->Start(); - } - - if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER)) { - if (!counters_sampler_started_.load(std::memory_order_acquire)) { - counters_sampler_ = new CountersSampler( - GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_SAMPLER))->GetBufferId(), - GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_SAMPLER))->GetId(), session_id_); - counters_sampler_started_.exchange(true, std::memory_order_release); - } - counters_sampler_->Start(); - } - - is_active_ = true; } + else + { + rocprofiler::queue::Queue::ResetSessionID(session_id_); + } + + if (FindFilterWithKind(ROCPROFILER_ATT_TRACE_COLLECTION) && !att_tracer_) + { + att_tracer_ = new att::AttTracer( + GetFilter(GetFilterIdWithKind(ROCPROFILER_ATT_TRACE_COLLECTION))->GetBufferId(), + GetFilter(GetFilterIdWithKind(ROCPROFILER_ATT_TRACE_COLLECTION))->GetId(), session_id_); + } + + if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION)) { + if (!spm_started_) + { + rocprofiler_spm_parameter_t* spmparameter = + GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetSpmParameterData(); + spmcounter_ = new spm::SpmCounters( + GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetBufferId(), + GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetId(), spmparameter, + session_id_); + } + if (!profiler_) + { + profiler_ = new profiler::Profiler( + GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetBufferId(), + GetFilter(GetFilterIdWithKind(ROCPROFILER_SPM_COLLECTION))->GetId(), session_id_); + } + startSpm(); + } + + if (FindFilterWithKind(ROCPROFILER_API_TRACE)) + { + std::vector domains = + GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetTraceData(); + if (!tracer_) + { + tracer_ = new tracer::Tracer( + session_id_, + (GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->HasCallback() + ? GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetCallback() + : nullptr), + GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetBufferId(), domains); + } + tracer_->StartRoctracer(); + } + + if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION)) + { + if (!pc_sampler_) + { + pc_sampler_ = new pc_sampler::PCSampler( + GetFilter(GetFilterIdWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION))->GetBufferId(), + GetFilter(GetFilterIdWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION))->GetId(), + session_id_); + } + pc_sampler_->Start(); + } + + if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER)) + { + if (!counters_sampler_) + { + counters_sampler_ = new CountersSampler( + GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_SAMPLER))->GetBufferId(), + GetFilter(GetFilterIdWithKind(ROCPROFILER_COUNTERS_SAMPLER))->GetId(), session_id_); + } + counters_sampler_->Start(); + } + + is_active_ = true; } -void Session::Terminate() { - if (is_active_) { - rocprofiler::queue::ResetSessionID(); - std::lock_guard lock(session_lock_); - if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION)) { - { stopSpm(); } - } - if (FindFilterWithKind(ROCPROFILER_API_TRACE)) { - std::vector domains = - GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetTraceData(); - if (tracer_started_.load(std::memory_order_acquire)) { - tracer_->StopRoctracer(); - } - } - if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION)) { - if (pc_sampler_started_.load(std::memory_order_acquire)) { - pc_sampler_->Stop(); - } - } +void Session::Terminate() +{ + if (!is_active_.load(std::memory_order_acquire)) + return; - if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER)) { - if (counters_sampler_started_.load(std::memory_order_acquire)) { - counters_sampler_->Stop(); - } - } + rocprofiler::queue::Queue::ResetSessionID(); + if (GetProfiler()) + GetProfiler()->WaitForPendingAndDestroy(); + if (GetAttTracer()) + GetAttTracer()->WaitForPendingAndDestroy(); - for (auto& buffer : *buffers_) { - buffer.second->Flush(); - } - is_active_ = false; + std::lock_guard lock(session_lock_); + if (FindFilterWithKind(ROCPROFILER_SPM_COLLECTION)) { + { stopSpm(); } } + if (FindFilterWithKind(ROCPROFILER_API_TRACE)) + { + std::vector domains = + GetFilter(GetFilterIdWithKind(ROCPROFILER_API_TRACE))->GetTraceData(); + if (tracer_) + tracer_->StopRoctracer(); + } + if (FindFilterWithKind(ROCPROFILER_PC_SAMPLING_COLLECTION) && pc_sampler_) + pc_sampler_->Stop(); + + if (FindFilterWithKind(ROCPROFILER_COUNTERS_SAMPLER) && counters_sampler_) + counters_sampler_->Stop(); + + for (auto& buffer : *buffers_) + buffer.second->Flush(); + + is_active_.store(false, std::memory_order_relaxed); } rocprofiler_session_id_t Session::GetId() { return session_id_; } @@ -352,7 +359,7 @@ void Session::DestroyBuffer(rocprofiler_buffer_id_t buffer_id) { rocprofiler_status_t Session::startSpm() { if (spmcounter_) { - spm_started_.exchange(true, std::memory_order_release); + spm_started_ = true; return spmcounter_->startSpm(); } else { std::cout << "Apply the SPM Filter" << std::endl; @@ -361,8 +368,8 @@ rocprofiler_status_t Session::startSpm() { } rocprofiler_status_t Session::stopSpm() { - if (spmcounter_ && spm_started_.load()) { - spm_started_.exchange(false, std::memory_order_release); + if (spmcounter_ && spm_started_) { + spm_started_ = false; return spmcounter_->stopSpm(); } else { std::cout << "SPM not started" << std::endl; diff --git a/src/core/session/session.h b/src/core/session/session.h index fb6f439f1f..6908bbc1b8 100644 --- a/src/core/session/session.h +++ b/src/core/session/session.h @@ -106,25 +106,19 @@ class Session { std::mutex filters_lock_; std::vector filters_; - std::atomic profiler_started_{false}; - std::atomic tracer_started_{false}; - std::atomic att_tracer_started_{false}; - att::AttTracer* att_tracer_; - std::atomic spm_started_{false}; + att::AttTracer* att_tracer_ = nullptr; + bool spm_started_{false}; - profiler::Profiler* profiler_; - tracer::Tracer* tracer_; - spm::SpmCounters* spmcounter_; + profiler::Profiler* profiler_ = nullptr; + tracer::Tracer* tracer_ = nullptr; + spm::SpmCounters* spmcounter_ = nullptr; - std::atomic pc_sampler_started_{false}; - pc_sampler::PCSampler* pc_sampler_; - - std::atomic counters_sampler_started_{false}; - CountersSampler* counters_sampler_; + pc_sampler::PCSampler* pc_sampler_ = nullptr; + CountersSampler* counters_sampler_ = nullptr; std::atomic buffers_counter_{1}; std::mutex buffers_lock_; - std::map* buffers_; + std::map* buffers_ = nullptr; std::atomic records_counter_{1};