From f5c6fc4dfafa2633b134d0a807240e31d31db153 Mon Sep 17 00:00:00 2001 From: Saleel Kudchadker Date: Tue, 7 Nov 2023 04:29:37 +0000 Subject: [PATCH] SWDEV-422207 - Report TS for Accumulate command Change-Id: Iba193a6068c1a2d25c2136643faee2c1e2591a07 --- rocclr/device/rocm/rocvirtual.cpp | 8 +++++++- rocclr/platform/activity.cpp | 18 ++++++++++++++---- rocclr/platform/activity.hpp | 1 + rocclr/platform/command.cpp | 2 ++ rocclr/platform/command.hpp | 24 ++++++++++++++++++------ 5 files changed, 42 insertions(+), 11 deletions(-) diff --git a/rocclr/device/rocm/rocvirtual.cpp b/rocclr/device/rocm/rocvirtual.cpp index 16706003f8..36658f3884 100644 --- a/rocclr/device/rocm/rocvirtual.cpp +++ b/rocclr/device/rocm/rocvirtual.cpp @@ -134,7 +134,8 @@ void Timestamp::checkGpuTime() { } // Avoid profiling data for the sync barrier, in tiny performance tests the first call // to ROCr is very slow and that also affects the overall performance of the callback thread - if (command().GetBatchHead() == nullptr || command().profilingInfo().marker_ts_) { + if (command().GetBatchHead() == nullptr || command().profilingInfo().marker_ts_ + || command().profilingInfo().multiple_ts_) { hsa_amd_profiling_dispatch_time_t time = {}; if (it->engine_ == HwQueueEngine::Compute) { hsa_amd_profiling_get_dispatch_time(gpu()->gpu_device(), it->signal_, &time); @@ -147,6 +148,11 @@ void Timestamp::checkGpuTime() { start = std::min(time.start, start); end = std::max(time.end, end); + + if (command().profilingInfo().multiple_ts_) { + command().AddTimeStamps(time.start, time.end); + } + ClPrint(amd::LOG_INFO, amd::LOG_SIG, "Signal = (0x%lx), start = %ld, " "end = %ld time taken= %ld ns", it->signal_.handle, time.start, time.end, time.end - time.start); diff --git a/rocclr/platform/activity.cpp b/rocclr/platform/activity.cpp index c29c997660..bbe142202d 100644 --- a/rocclr/platform/activity.cpp +++ b/rocclr/platform/activity.cpp @@ -50,19 +50,19 @@ bool IsEnabled(OpId operation_id) { } void ReportActivity(const amd::Command& command) { - assert(command.profilingInfo().enabled_ && "profiling must be enabled for this command"); + assert(command.profilingInfo().enabled_ && "Profiling must be enabled for this command"); activity_op_t operation_id = OperationId(command.type()); - if (operation_id >= OP_ID_NUMBER) + if (operation_id >= OP_ID_NUMBER) { // This command does not translate into a profiler activity (dispatch, memcopy, etc...), there // is nothing to report to the profiler. return; + } auto function = report_activity.load(std::memory_order_relaxed); if (!function) return; const auto* queue = command.queue(); assert(queue != nullptr); - activity_record_t record{ ACTIVITY_DOMAIN_HIP_OPS, // activity domain command.type(), // activity kind @@ -101,7 +101,17 @@ void ReportActivity(const amd::Command& command) { break; } - function(ACTIVITY_DOMAIN_HIP_OPS, operation_id, &record); + if (command.profilingInfo().tsList_.size() > 0) { + for (auto& it : command.profilingInfo().tsList_) { + record.begin_ns = it.first; + record.end_ns = it.second; + function(ACTIVITY_DOMAIN_HIP_OPS, operation_id, &record); + } + } else { + record.begin_ns = command.profilingInfo().start_; + record.end_ns = command.profilingInfo().end_; + function(ACTIVITY_DOMAIN_HIP_OPS, operation_id, &record); + } } } // namespace activity_prof diff --git a/rocclr/platform/activity.hpp b/rocclr/platform/activity.hpp index 46c8852aaa..873e48a7b2 100644 --- a/rocclr/platform/activity.hpp +++ b/rocclr/platform/activity.hpp @@ -50,6 +50,7 @@ extern __declspec(thread) activity_correlation_id_t correlation_id; constexpr OpId OperationId(cl_command_type commandType) { switch (commandType) { case CL_COMMAND_NDRANGE_KERNEL: + case CL_COMMAND_TASK: return OP_ID_DISPATCH; case CL_COMMAND_READ_BUFFER: case CL_COMMAND_READ_BUFFER_RECT: diff --git a/rocclr/platform/command.cpp b/rocclr/platform/command.cpp index 0810d998e0..052b1f09e5 100644 --- a/rocclr/platform/command.cpp +++ b/rocclr/platform/command.cpp @@ -417,6 +417,8 @@ NDRangeKernelCommand::NDRangeKernelCommand(HostQueue& queue, const EventWaitList if (cooperativeGroups()) { setNumWorkgroups(); } + + // This optimization will set marker_ts_ but may not submit a batch. if (forceProfiling) { profilingInfo_.enabled_ = true; profilingInfo_.clear(); diff --git a/rocclr/platform/command.hpp b/rocclr/platform/command.hpp index ece4a21a0b..f5f6e92554 100644 --- a/rocclr/platform/command.hpp +++ b/rocclr/platform/command.hpp @@ -104,7 +104,8 @@ class Event : public RuntimeObject { static const EventWaitList nullWaitList; struct ProfilingInfo { - ProfilingInfo(bool enabled = false) : enabled_(enabled), marker_ts_(false) { + ProfilingInfo(bool enabled = false) + : enabled_(enabled), marker_ts_(false), multiple_ts_(false) { if (enabled) { clear(); correlation_id_ = activity_prof::correlation_id; @@ -115,15 +116,19 @@ class Event : public RuntimeObject { uint64_t submitted_; uint64_t start_; uint64_t end_; - uint64_t correlation_id_; - bool enabled_; //!< Profiling enabled for the wave limiter - bool marker_ts_; //!< TS marker + std::vector> tsList_; - void clear() { + uint64_t correlation_id_; + bool enabled_; //!< Profiling enabled for the wave limiter + bool marker_ts_; //!< TS marker + bool multiple_ts_; //!< Multiple TS + + void clear() { queued_ = 0ULL; submitted_ = 0ULL; start_ = 0ULL; end_ = 0ULL; + tsList_.clear(); } } profilingInfo_; @@ -220,6 +225,11 @@ class Event : public RuntimeObject { //! Set release scope for the event void setEventScope(int32_t scope) { event_scope_ = scope; } + + //! Add a timestamp to the list + void AddTimeStamps(uint64_t start, uint64_t end) { + profilingInfo_.tsList_.push_back(std::make_pair(start, end)); + } }; union CopyMetadata { @@ -1255,7 +1265,9 @@ class AccumulateCommand : public Command { //! Create a new Marker AccumulateCommand(HostQueue& queue, const EventWaitList& eventWaitList = nullWaitList, const Event* waitingEvent = nullptr) - : Command(queue, CL_COMMAND_TASK, eventWaitList, 0, waitingEvent) {} + : Command(queue, CL_COMMAND_TASK, eventWaitList, 0, waitingEvent) { + profilingInfo_.multiple_ts_ = true; + } //! The command implementation virtual void submit(device::VirtualDevice& device) {