SWDEV-422207 - Report TS for Accumulate command

Change-Id: Iba193a6068c1a2d25c2136643faee2c1e2591a07

rocclr/device/rocm/rocvirtual.cpp

@@ -134,7 +134,8 @@ void Timestamp::checkGpuTime() {
       }
       // Avoid profiling data for the sync barrier, in tiny performance tests the first call
       // to ROCr is very slow and that also affects the overall performance of the callback thread
-      if (command().GetBatchHead() == nullptr || command().profilingInfo().marker_ts_) {
+      if (command().GetBatchHead() == nullptr || command().profilingInfo().marker_ts_
+          || command().profilingInfo().multiple_ts_) {
         hsa_amd_profiling_dispatch_time_t time = {};
         if (it->engine_ == HwQueueEngine::Compute) {
           hsa_amd_profiling_get_dispatch_time(gpu()->gpu_device(), it->signal_, &time);
@@ -147,6 +148,11 @@ void Timestamp::checkGpuTime() {
 
         start = std::min(time.start, start);
         end = std::max(time.end, end);
+
+        if (command().profilingInfo().multiple_ts_) {
+          command().AddTimeStamps(time.start, time.end);
+        }
+
         ClPrint(amd::LOG_INFO, amd::LOG_SIG, "Signal = (0x%lx), start = %ld, "
           "end = %ld time taken= %ld ns", it->signal_.handle, time.start, time.end,
           time.end - time.start);

rocclr/platform/activity.cpp

@@ -50,19 +50,19 @@ bool IsEnabled(OpId operation_id) {
 }
 
 void ReportActivity(const amd::Command& command) {
-  assert(command.profilingInfo().enabled_ && "profiling must be enabled for this command");
+  assert(command.profilingInfo().enabled_ && "Profiling must be enabled for this command");
   activity_op_t operation_id = OperationId(command.type());
-  if (operation_id >= OP_ID_NUMBER)
+  if (operation_id >= OP_ID_NUMBER) {
     // This command does not translate into a profiler activity (dispatch, memcopy, etc...), there
     // is nothing to report to the profiler.
     return;
+  }
 
   auto function = report_activity.load(std::memory_order_relaxed);
   if (!function) return;
 
   const auto* queue = command.queue();
   assert(queue != nullptr);
-
   activity_record_t record{
       ACTIVITY_DOMAIN_HIP_OPS,                  // activity domain
       command.type(),                           // activity kind
@@ -101,7 +101,17 @@ void ReportActivity(const amd::Command& command) {
       break;
   }
 
-  function(ACTIVITY_DOMAIN_HIP_OPS, operation_id, &record);
+  if (command.profilingInfo().tsList_.size() > 0) {
+    for (auto& it : command.profilingInfo().tsList_) {
+      record.begin_ns = it.first;
+      record.end_ns = it.second;
+      function(ACTIVITY_DOMAIN_HIP_OPS, operation_id, &record);
+    }
+  } else {
+      record.begin_ns = command.profilingInfo().start_;
+      record.end_ns = command.profilingInfo().end_;
+      function(ACTIVITY_DOMAIN_HIP_OPS, operation_id, &record);
+  }
 }
 
 }  // namespace activity_prof

rocclr/platform/activity.hpp

@@ -50,6 +50,7 @@ extern __declspec(thread) activity_correlation_id_t correlation_id;
 constexpr OpId OperationId(cl_command_type commandType) {
   switch (commandType) {
     case CL_COMMAND_NDRANGE_KERNEL:
+    case CL_COMMAND_TASK:
       return OP_ID_DISPATCH;
     case CL_COMMAND_READ_BUFFER:
     case CL_COMMAND_READ_BUFFER_RECT:

rocclr/platform/command.cpp

@@ -417,6 +417,8 @@ NDRangeKernelCommand::NDRangeKernelCommand(HostQueue& queue, const EventWaitList
   if (cooperativeGroups()) {
     setNumWorkgroups();
   }
+
+  // This optimization will set marker_ts_ but may not submit a batch.
   if (forceProfiling) {
     profilingInfo_.enabled_ = true;
     profilingInfo_.clear();

rocclr/platform/command.hpp

@@ -104,7 +104,8 @@ class Event : public RuntimeObject {
   static const EventWaitList nullWaitList;
 
   struct ProfilingInfo {
-    ProfilingInfo(bool enabled = false) : enabled_(enabled), marker_ts_(false) {
+    ProfilingInfo(bool enabled = false)
+      : enabled_(enabled), marker_ts_(false), multiple_ts_(false) {
       if (enabled) {
         clear();
         correlation_id_ = activity_prof::correlation_id;
@@ -115,15 +116,19 @@ class Event : public RuntimeObject {
     uint64_t submitted_;
     uint64_t start_;
     uint64_t end_;
-    uint64_t correlation_id_;
-    bool enabled_;    //!< Profiling enabled for the wave limiter
-    bool marker_ts_;  //!< TS marker
+    std::vector<std::pair<uint64_t, uint64_t>> tsList_;
 
-    void clear() {
+    uint64_t correlation_id_;
+    bool enabled_;        //!< Profiling enabled for the wave limiter
+    bool marker_ts_;      //!< TS marker
+    bool multiple_ts_;    //!< Multiple TS
+
+   void clear() {
       queued_ = 0ULL;
       submitted_ = 0ULL;
       start_ = 0ULL;
       end_ = 0ULL;
+      tsList_.clear();
     }
   } profilingInfo_;
 
@@ -220,6 +225,11 @@ class Event : public RuntimeObject {
 
   //! Set release scope for the event
   void setEventScope(int32_t scope) { event_scope_ = scope; }
+
+  //! Add a timestamp to the list
+  void AddTimeStamps(uint64_t start, uint64_t end) {
+    profilingInfo_.tsList_.push_back(std::make_pair(start, end));
+  }
 };
 
 union CopyMetadata {
@@ -1255,7 +1265,9 @@ class AccumulateCommand : public Command {
   //! Create a new Marker
   AccumulateCommand(HostQueue& queue, const EventWaitList& eventWaitList = nullWaitList,
          const Event* waitingEvent = nullptr)
-      : Command(queue, CL_COMMAND_TASK, eventWaitList, 0, waitingEvent) {}
+      : Command(queue, CL_COMMAND_TASK, eventWaitList, 0, waitingEvent) {
+        profilingInfo_.multiple_ts_ = true;
+      }
 
   //! The command implementation
   virtual void submit(device::VirtualDevice& device) {