SWDEV-290384 - Disable HSA callback for any host wait

Change-Id: Ie876deb62859f5551f4ed69eb8187ac3fa35f42a


[ROCm/clr commit: c144383971]

projects/clr/rocclr/device/rocm/rocvirtual.cpp

@@ -403,7 +403,8 @@ hsa_signal_t VirtualGPU::HwQueueTracker::ActiveSignal(
     ts->AddProfilingSignal(prof_signal);
     // If direct dispatch is enabled and the batch head isn't null, then it's a marker and
     // requires the batch update upon HSA signal completion
-    if (AMD_DIRECT_DISPATCH && (ts->command().GetBatchHead() != nullptr)) {
+    if (AMD_DIRECT_DISPATCH && (ts->command().GetBatchHead() != nullptr) &&
+        !ts->command().CpuWaitRequested()) {
       uint32_t init_value = kInitSignalValueOne;
       // If API callback is enabled, then use a blocking signal for AQL queue.
       // HSA signal will be acquired in SW and released after HSA signal callback
@@ -421,6 +422,9 @@ hsa_signal_t VirtualGPU::HwQueueTracker::ActiveSignal(
         ClPrint(amd::LOG_INFO, amd::LOG_SIG, "Set Handler: handle(0x%lx), timestamp(%p)",
           prof_signal->signal_.handle, prof_signal);
       }
+      // Update the current command/marker with HW event
+      prof_signal->retain();
+      ts->command().SetHwEvent(prof_signal);
     }
     if (!sdma_profiling_) {
       hsa_amd_profiling_async_copy_enable(true);
@@ -1253,13 +1257,6 @@ void VirtualGPU::profilingEnd(amd::Command& command) {
       timestamp_->end();
     }
     command.setData(timestamp_);
-
-    // Update HW event only for batches
-    if ((AMD_DIRECT_DISPATCH) && (command.GetBatchHead() != nullptr)) {
-      timestamp_->Signals().back()->retain();
-      command.SetHwEvent(timestamp_->Signals().back());
-    }
-
     timestamp_ = nullptr;
   }
 }
@@ -2949,7 +2946,7 @@ void VirtualGPU::submitMarker(amd::Marker& vcmd) {
   if (AMD_DIRECT_DISPATCH || vcmd.profilingInfo().marker_ts_) {
     // Make sure VirtualGPU has an exclusive access to the resources
     amd::ScopedLock lock(execution());
-    if (vcmd.CpuWaitRequested() && hasPendingDispatch_ == false) {
+    if (vcmd.CpuWaitRequested()) {
       // It should be safe to call flush directly if there are not pending dispatches without
       // HSA signal callback
       flush(vcmd.GetBatchHead());

projects/clr/rocclr/device/rocm/rocvirtual.hpp

@@ -89,7 +89,7 @@ class Timestamp : public amd::HeapObject {
   uint64_t    start_;
   uint64_t    end_;
   VirtualGPU* gpu_;               //!< Virtual GPU, associated with this timestamp
-  const amd::Command& command_;   //!< Command, associated with this timestamp
+  amd::Command& command_;         ///!< Command, associated with this timestamp
   amd::Command* parsedCommand_;   //!< Command down the list, considering command_ as head
   std::vector<ProfilingSignal*> signals_; //!< The list of all signals, associated with the TS
   hsa_signal_t callback_signal_;  //!< Signal associated with a callback for possible later update
@@ -98,7 +98,7 @@ class Timestamp : public amd::HeapObject {
   Timestamp& operator=(const Timestamp&) = delete;
 
  public:
-  Timestamp(VirtualGPU* gpu, const amd::Command& command)
+  Timestamp(VirtualGPU* gpu, amd::Command& command)
     : start_(std::numeric_limits<uint64_t>::max())
     , end_(0)
     , gpu_(gpu)
@@ -144,7 +144,7 @@ class Timestamp : public amd::HeapObject {
   static double getGpuTicksToTime() { return ticksToTime_; }
 
   //! Returns amd::command assigned to this timestamp
-  const amd::Command& command() const { return command_; }
+  amd::Command& command() const { return command_; }
 
   //! Sets the parsed command
   void setParsedCommand(amd::Command* command) { parsedCommand_ = command; }

projects/clr/rocclr/platform/command.hpp

@@ -242,6 +242,8 @@ class Command : public Event {
   const Event* waitingEvent_;     //!< Waiting event associated with the marker
 
  protected:
+  bool cpu_wait_ = false;         //!< If true, then the command was issued for CPU/GPU sync
+
   //! The Events that need to complete before this command is submitted.
   EventWaitList eventWaitList_;
 
@@ -336,6 +338,9 @@ class Command : public Event {
   Command* GetBatchHead() const { return batch_head_; }
 
   const Event* waitingEvent() const { return waitingEvent_; }
+
+  //! Check if this command(should be a marker) requires CPU wait
+  bool CpuWaitRequested() const { return cpu_wait_; }
 };
 
 class UserEvent : public Command {
@@ -998,22 +1003,14 @@ class ExternalSemaphoreCmd : public Command {
 
 
 class Marker : public Command {
- private:
-  bool cpu_wait_;   //!< If true, then the marker was issued for CPU/GPU sync
-
  public:
   //! Create a new Marker
   Marker(HostQueue& queue, bool userVisible, const EventWaitList& eventWaitList = nullWaitList,
          const Event* waitingEvent = nullptr, bool cpu_wait = false)
-      : Command(queue, userVisible ? CL_COMMAND_MARKER : 0, eventWaitList, 0, waitingEvent)
-      , cpu_wait_(cpu_wait) {}
+      : Command(queue, userVisible ? CL_COMMAND_MARKER : 0, eventWaitList, 0, waitingEvent) { cpu_wait_ = cpu_wait; }
 
   //! The actual command implementation.
   virtual void submit(device::VirtualDevice& device) { device.submitMarker(*this); }
-
-  //! Check if this marker requires CPU wait
-  bool CpuWaitRequested() const { return cpu_wait_; }
-
 };
 
 /*! \brief  Maps CL objects created from external ones and syncs the contents (blocking).