Separate blit compute interface from queue creation

The runtime needs a queue on which to submit cache management commands.
Device-to-device blit copy already creates a queue unconditionally.
We can share this queue for both purposes.

This change restructures the BlitKernel interface to accept, rather than
create, a queue. GpuAgent creates queues as needed for both cache
management and blit compute.

Fix queue full detection in AcquireWriteIndex (<= vs <).

Change-Id: I61d0c6b9d04f2dba74872f0676ad791435778ba4


[ROCm/ROCR-Runtime commit: f7ab361347]

projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_kernel.h

@@ -51,7 +51,7 @@
 namespace amd {
 class BlitKernel : public core::Blit {
  public:
-  explicit BlitKernel();
+  explicit BlitKernel(core::Queue* queue);
   virtual ~BlitKernel() override;
 
   /// @brief Initialize a blit kernel object.
@@ -175,12 +175,9 @@ class BlitKernel : public core::Blit {
   std::map<KernelType, KernelCode> kernels_;
 
   /// AQL queue for submitting the vector copy kernel.
-  hsa_queue_t* queue_;
+  core::Queue* queue_;
   uint32_t queue_bitmask_;
 
-  /// Index to track concurrent kernel launch.
-  volatile uint64_t cached_index_;
-
   /// Pointer to the kernel argument buffer.
   KernelArgs* kernarg_async_;
   uint32_t kernarg_async_mask_;

projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h

@@ -79,10 +79,11 @@ class GpuAgentInt : public core::Agent {
   // @retval HSA_STATUS_SUCCESS DMA queue initialization is successful.
   virtual void InitDma() = 0;
 
-  // @brief Initialize blit kernel object based on AQL queue.
+  // @brief Initialization hook invoked after tools library has loaded,
+  // to allow tools interception of interface functions.
   //
-  // @retval HSA_STATUS_SUCCESS blit kernel object initialization is successful.
-  virtual hsa_status_t InitBlitKernel() = 0;
+  // @retval HSA_STATUS_SUCCESS if initialization is successful.
+  virtual hsa_status_t PostToolsInit() = 0;
 
   // @brief Invoke the user provided callback for each region accessible by
   // this agent.
@@ -184,7 +185,7 @@ class GpuAgent : public GpuAgentInt {
   void InitDma() override;
 
   // @brief Override from core::Agent.
-  hsa_status_t InitBlitKernel() override;
+  hsa_status_t PostToolsInit() override;
 
   uint16_t GetMicrocodeVersion() const;
 
@@ -309,15 +310,18 @@ class GpuAgent : public GpuAgentInt {
   static const uint32_t minAqlSize_ = 0x1000;   // 4KB min
   static const uint32_t maxAqlSize_ = 0x20000;  // 8MB max
 
+  // @brief Create a queue through HSA API to allow tools to intercept.
+  core::Queue* CreateInterceptibleQueue();
+
   // @brief Create SDMA blit object.
   //
   // @retval NULL if SDMA blit creation and initialization failed.
   core::Blit* CreateBlitSdma();
 
-  // @brief Create Kernel blit object.
+  // @brief Create Kernel blit object using provided compute queue.
   //
   // @retval NULL if Kernel blit creation and initialization failed.
-  core::Blit* CreateBlitKernel();
+  core::Blit* CreateBlitKernel(core::Queue* queue);
 
   // @brief Invoke the user provided callback for every region in @p regions.
   //
@@ -359,15 +363,19 @@ class GpuAgent : public GpuAgentInt {
   // @brief Default scratch size per work item.
   size_t scratch_per_thread_;
 
-  // @brief Blit object to handle memory copy from system to device memory.
-  core::Blit* blit_h2d_;
+  // @brief Blit interfaces for each data path.
+  enum BlitEnum { BlitHostToDev, BlitDevToHost, BlitDevToDev, BlitCount };
 
-  // @brief Blit object to handle memory copy from device to system memory.
-  core::Blit* blit_d2h_;
+  core::Blit* blits_[BlitCount];
 
-  // @brief Blit object to handle memory copy from device to device memory, and
-  // memory fill.
-  core::Blit* blit_d2d_;
+  // @brief AQL queues for cache management and blit compute usage.
+  enum QueueEnum {
+    QueueUtility,   // Cache management and device to {host,device} blit compute
+    QueueBlitOnly,  // Host to device blit
+    QueueCount
+  };
+
+  core::Queue* queues_[QueueCount];
 
   // @brief Mutex to protect the update to coherency type.
   KernelMutex coherency_lock_;