SWDEV-335780 - Indicate if handler is queued

Maintain status of handler callback. For event records we no longer
submit callbacks to reduce the load on the async handler thread. However
without a callback we leak command memory/decrement refcounts. Indicate
status of the handler which we can use to queue a callback when
finish is called.

Change-Id: I89fd02f3d047a0e8162664ee17581a14795f1928


[ROCm/clr commit: 5df34a2f7a]

projects/clr/rocclr/device/device.hpp

@@ -1258,6 +1258,9 @@ class VirtualDevice : public amd::HeapObject {
   //! Returns true if device has active wait setting
   bool ActiveWait() const;
 
+  //! Returns the status of queue handler callback
+  virtual bool isHandlerPending() const = 0;
+
  private:
   //! Disable default copy constructor
   VirtualDevice& operator=(const VirtualDevice&);

projects/clr/rocclr/device/pal/palvirtual.hpp

@@ -358,6 +358,8 @@ class VirtualGPU : public device::VirtualDevice {
 
   void profilerAttach(bool enable = false) {}
 
+  bool isHandlerPending() const { return false; }
+
   //! Returns GPU device object associated with this kernel
   const Device& dev() const { return gpuDevice_; }
 

projects/clr/rocclr/device/rocm/rocdevice.cpp

@@ -2659,7 +2659,7 @@ void Device::getHwEventTime(const amd::Event& event, uint64_t* start, uint64_t*
 // ================================================================================================
 bool Device::IsCacheFlushed(Device::CacheState state) const {
 
-  return ROC_EVENT_NO_FLUSH ? 
+  return ROC_EVENT_NO_FLUSH ?
          (static_cast<int>(state) == cache_state_.load(std::memory_order_relaxed)) : true;
 }
 

projects/clr/rocclr/device/rocm/rocvirtual.cpp

@@ -446,11 +446,12 @@ hsa_signal_t VirtualGPU::HwQueueTracker::ActiveSignal(
           ClPrint(amd::LOG_INFO, amd::LOG_SIG, "Set Handler: handle(0x%lx), timestamp(%p)",
             prof_signal->signal_.handle, prof_signal);
         }
-
+        SetHandlerPending(false);
         // Update the current command/marker with HW event
         prof_signal->retain();
         ts->command().SetHwEvent(prof_signal);
       } else if (ts->command().profilingInfo().marker_ts_) {
+        SetHandlerPending(true);
         // Update the current command/marker with HW event
         prof_signal->retain();
         ts->command().SetHwEvent(prof_signal);

projects/clr/rocclr/device/rocm/rocvirtual.hpp

@@ -213,7 +213,7 @@ class VirtualGPU : public device::VirtualDevice {
 
   class HwQueueTracker : public amd::EmbeddedObject {
    public:
-    HwQueueTracker(const VirtualGPU& gpu): gpu_(gpu) {}
+    HwQueueTracker(const VirtualGPU& gpu): gpu_(gpu), handlerPending_(false) {}
 
     ~HwQueueTracker();
 
@@ -255,6 +255,11 @@ class VirtualGPU : public device::VirtualDevice {
     //! Empty check for external signals
     bool IsExternalSignalListEmpty() const { return external_signals_.empty(); }
 
+    //! Set the status to indicate a pending handler
+    void SetHandlerPending(bool pending) { handlerPending_ = pending; }
+
+    //! Check if callback has been queued
+    bool IsHandlerPending() const { return handlerPending_; }
   private:
     //! Wait for the next active signal
     void WaitNext() {
@@ -273,6 +278,7 @@ class VirtualGPU : public device::VirtualDevice {
     const VirtualGPU& gpu_;       //!< VirtualGPU, associated with this tracker
     std::vector<ProfilingSignal*> external_signals_;  //!< External signals for a wait in this queue
     std::vector<hsa_signal_t> waiting_signals_;   //!< Current waiting signals in this queue
+    bool handlerPending_;         //!< This indicates if we have queued a callback handler
   };
 
   VirtualGPU(Device& device, bool profiling = false, bool cooperative = false,
@@ -393,6 +399,10 @@ class VirtualGPU : public device::VirtualDevice {
   void profilerAttach(bool enable = false) { profilerAttached_ = enable; }
 
   bool isProfilerAttached() const { return profilerAttached_; }
+
+  //! Indicates the status of the callback handler. The callback would process the commands
+  //! and would collect profiling data, update refcounts
+  bool isHandlerPending() const { return barriers_.IsHandlerPending(); }
   // } roc OpenCL integration
  private:
   //! Dispatches a barrier with blocking HSA signals

projects/clr/rocclr/platform/commandqueue.cpp

@@ -114,8 +114,7 @@ void HostQueue::finish() {
       return;
     }
   }
-
-  if (nullptr == command || !isCacheFlushed) {
+  if (nullptr == command || !isCacheFlushed || vdev()->isHandlerPending()) {
     if (nullptr != command) {
       command->release();
     }