SWDEV-439628 - hipGraphExecKernelNodeSetParams to update graph kernel node params with graph performance optimizations.

During hipGraphExecKernelNodeSetParams kernel function can also be updated.
Hence size required for kernel parameters differs from what is allocated during graphInstantiation.
So, create new 128KB kernel pool and allocate kernel args from the pool.
If the pool is full create new 128KB pool. Release kernel pools when graph exec object is destroyed.

Change-Id: I9567946d63400c79cbfd4c5439c654c92557ceae

hipamd/src/hip_context.cpp

@@ -90,7 +90,7 @@ void setCurrentDevice(unsigned int index) {
 
 hip::Stream* getStream(hipStream_t stream, bool wait) {
  if (stream == nullptr) {
-    return getNullStream();
+    return getNullStream(wait);
   } else {
     hip::Stream* hip_stream = reinterpret_cast<hip::Stream*>(stream);
     if (wait && !(hip_stream->Flags() & hipStreamNonBlocking)) {
@@ -128,9 +128,9 @@ int getDeviceID(amd::Context& ctx) {
 }
 
 // ================================================================================================
-hip::Stream* getNullStream() {
+hip::Stream* getNullStream(bool wait ) {
   Device* device = getCurrentDevice();
-  return device ? device->NullStream() : nullptr;
+  return device ? device->NullStream(wait) : nullptr;
 }
 
 hipError_t hipInit(unsigned int flags) {

hipamd/src/hip_device.cpp

@@ -30,7 +30,7 @@
 namespace hip {
 
 // ================================================================================================
-hip::Stream* Device::NullStream() {
+hip::Stream* Device::NullStream(bool wait) {
   if (null_stream_ == nullptr) {
     null_stream_ = new Stream(this, Stream::Priority::Normal, 0, true);
   }
@@ -38,8 +38,10 @@ hip::Stream* Device::NullStream() {
   if (null_stream_ == nullptr) {
     return nullptr;
   }
-  // Wait for all active streams before executing commands on the default
-  iHipWaitActiveStreams(null_stream_);
+  if (wait == true) {
+    // Wait for all active streams before executing commands on the default
+    iHipWaitActiveStreams(null_stream_);
+  }
   return null_stream_;
 }
 

hipamd/src/hip_graph.cpp

@@ -1552,7 +1552,15 @@ hipError_t hipGraphExecKernelNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNo
   if (clonedNode == nullptr) {
     HIP_RETURN(hipErrorInvalidValue);
   }
-  HIP_RETURN(reinterpret_cast<hip::GraphKernelNode*>(clonedNode)->SetParams(pNodeParams));
+  hipError_t status = reinterpret_cast<hip::GraphKernelNode*>(clonedNode)->SetParams(pNodeParams);
+  if(status != hipSuccess) {
+    HIP_RETURN(status);
+  }
+  if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) {
+    status = reinterpret_cast<hip::GraphExec*>(hGraphExec)
+        ->UpdateAQLPacket(reinterpret_cast<hip::GraphKernelNode*>(clonedNode));
+  }
+  HIP_RETURN(status);
 }
 
 hipError_t hipGraphChildGraphNodeGetGraph(hipGraphNode_t node, hipGraph_t* pGraph) {

hipamd/src/hip_graph_internal.cpp

@@ -332,6 +332,9 @@ hipError_t GraphExec::CreateStreams(uint32_t num_streams) {
     }
     parallel_streams_.push_back(stream);
   }
+  // Don't wait for other streams to finish.
+  // Capture stream is to capture AQL packet.
+  capture_stream_ = hip::getNullStream(false);
   return hipSuccess;
 }
 
@@ -353,13 +356,10 @@ hipError_t GraphExec::CaptureAQLPackets() {
   hipError_t status = hipSuccess;
   if (parallelLists_.size() == 1) {
     size_t kernArgSizeForGraph = 0;
-    hip::Stream* stream = nullptr;
     // GPU packet capture is enabled for kernel nodes. Calculate the kernel
     // arg size required for all graph kernel nodes to allocate
     for (const auto& list : parallelLists_) {
-      stream = GetAvailableStreams();
       for (auto& node : list) {
-        node->SetStream(stream, this);
         if (node->GetType() == hipGraphNodeTypeKernel) {
           kernArgSizeForGraph += reinterpret_cast<hip::GraphKernelNode*>(node)->GetKerArgSize();
         }
@@ -386,7 +386,6 @@ hipError_t GraphExec::CaptureAQLPackets() {
     for (auto& node : topoOrder_) {
       if (node->GetType() == hipGraphNodeTypeKernel) {
         auto kernelNode = reinterpret_cast<hip::GraphKernelNode*>(node);
-        status = node->CreateCommand(node->GetQueue());
         // From the kernel pool allocate the kern arg size required for the current kernel node.
         address kernArgOffset = allocKernArg(kernelNode->GetKernargSegmentByteSize(),
                                              kernelNode->GetKernargSegmentAlignment());
@@ -394,7 +393,7 @@ hipError_t GraphExec::CaptureAQLPackets() {
           return hipErrorMemoryAllocation;
         }
         // Form GPU packet capture for the kernel node.
-        kernelNode->CaptureAndFormPacket(kernArgOffset);
+        kernelNode->CaptureAndFormPacket(capture_stream_, kernArgOffset) ;
       }
     }
 
@@ -408,7 +407,7 @@ hipError_t GraphExec::CaptureAQLPackets() {
         address dev_ptr = kernarg_pool_graph_ + kernarg_pool_size_graph_ - sizeof(int);
         *dev_ptr = host_val;
         if (device->info().hdpMemFlushCntl == nullptr) {
-          amd::Command* command = new amd::Marker(*stream, true);
+          amd::Command* command = new amd::Marker(*capture_stream_, true);
           if (command != nullptr) {
             command->enqueue();
             command->release();
@@ -426,6 +425,50 @@ hipError_t GraphExec::CaptureAQLPackets() {
   return status;
 }
 
+hipError_t GraphExec::UpdateAQLPacket(hip::GraphKernelNode* node) {
+  if (parallelLists_.size() == 1) {
+    size_t pool_new_usage = 0;
+    address result = nullptr;
+    if (!kernarg_graph_.empty()) {
+      // 1. Allocate memory for the kernel args
+      size_t kernArgSizeForNode = 0;
+      kernArgSizeForNode = node->GetKerArgSize();
+
+      result = amd::alignUp(kernarg_graph_.back() + kernarg_graph_cur_offset_,
+                            node->GetKernargSegmentAlignment());
+      pool_new_usage = (result + kernArgSizeForNode) - kernarg_graph_.back();
+    }
+    if (pool_new_usage != 0 && pool_new_usage <= kernarg_graph_size_) {
+      kernarg_graph_cur_offset_ = pool_new_usage;
+    } else {
+      address kernarg_graph;
+      auto device = g_devices[ihipGetDevice()]->devices()[0];
+      if (device->info().largeBar_) {
+        kernarg_graph = reinterpret_cast<address>(device->deviceLocalAlloc(kernarg_graph_size_));
+      } else {
+        kernarg_graph = reinterpret_cast<address>(
+            device->hostAlloc(kernarg_graph_size_, 0, amd::Device::MemorySegment::kKernArg));
+      }
+      kernarg_graph_.push_back(kernarg_graph);
+      kernarg_graph_cur_offset_ = 0;
+
+      // 1. Allocate memory for the kernel args
+      size_t kernArgSizeForNode = 0;
+      kernArgSizeForNode = node->GetKerArgSize();
+      result = amd::alignUp(kernarg_graph_.back() + kernarg_graph_cur_offset_,
+                            node->GetKernargSegmentAlignment());
+      const size_t pool_new_usage = (result + kernArgSizeForNode) - kernarg_graph_.back();
+      if (pool_new_usage <= kernarg_graph_size_) {
+        kernarg_graph_cur_offset_ = pool_new_usage;
+      }
+    }
+
+    // 2. copy kernel args / create new AQL packet
+    node->CaptureAndFormPacket(capture_stream_, result);
+  }
+  return hipSuccess;
+}
+
 hipError_t FillCommands(std::vector<std::vector<Node>>& parallelLists,
                         std::unordered_map<Node, std::vector<Node>>& nodeWaitLists,
                         std::vector<Node>& topoOrder, Graph* clonedGraph,

hipamd/src/hip_graph_internal.hpp

@@ -543,7 +543,7 @@ struct Graph {
     graphInstantiated_ = graphInstantiate;
   }
 };
-
+struct GraphKernelNode;
 struct GraphExec {
   std::vector<std::vector<Node>> parallelLists_;
   // Topological order of the graph doesn't include nodes embedded as part of the child graph
@@ -551,6 +551,7 @@ struct GraphExec {
   std::unordered_map<Node, std::vector<Node>> nodeWaitLists_;
   struct Graph* clonedGraph_;
   std::vector<hip::Stream*> parallel_streams_;
+  hip::Stream* capture_stream_;
   uint currentQueueIndex_;
   std::unordered_map<Node, Node> clonedNodes_;
   amd::Command* lastEnqueuedCommand_;
@@ -563,6 +564,10 @@ struct GraphExec {
   address kernarg_pool_graph_ = nullptr;
   uint32_t kernarg_pool_size_graph_ = 0;
   uint32_t kernarg_pool_cur_graph_offset_ = 0;
+  std::vector<address> kernarg_graph_;
+  uint32_t kernarg_graph_cur_offset_ = 0;
+  uint32_t kernarg_graph_size_ = 128 * Ki;
+
  public:
   GraphExec(std::vector<Node>& topoOrder, std::vector<std::vector<Node>>& lists,
             std::unordered_map<Node, std::vector<Node>>& nodeWaitLists, struct Graph*& clonedGraph,
@@ -591,6 +596,9 @@ struct GraphExec {
     auto device = g_devices[ihipGetDevice()]->devices()[0];
     if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) {
       device->hostFree(kernarg_pool_graph_, kernarg_pool_size_graph_);
+      for (auto& element : kernarg_graph_) {
+        device->hostFree(element, kernarg_graph_size_);
+      }
     }
     amd::ScopedLock lock(graphExecSetLock_);
     graphExecSet_.erase(this);
@@ -636,6 +644,7 @@ struct GraphExec {
   hipError_t Run(hipStream_t stream);
   // Capture GPU Packets from graph commands
   hipError_t CaptureAQLPackets();
+  hipError_t UpdateAQLPacket(hip::GraphKernelNode* node);
 };
 
 struct ChildGraphNode : public GraphNode {
@@ -793,19 +802,20 @@ class GraphKernelNode : public GraphNode {
     out << "];";
     }
 
-    void CaptureAndFormPacket(address kernArgOffset) {
-      for (auto& command : commands_) {
-        reinterpret_cast<amd::NDRangeKernelCommand*>(command)->setCapturingState(
-            true, GetAqlPacket(), kernArgOffset);
+  void CaptureAndFormPacket(hip::Stream* capture_stream, address kernArgOffset) {
+    hipError_t status = CreateCommand(capture_stream);
+    for (auto& command : commands_) {
+      reinterpret_cast<amd::NDRangeKernelCommand*>(command)->setCapturingState(
+          true, GetAqlPacket(), kernArgOffset);
 
-        // Enqueue command to capture GPU Packet. The packet is not submitted to the device.
-        // The packet is stored in gpuPacket_ and submitted during graph launch.
-        command->submit(*(command->queue())->vdev());
-        // Need to ensure if the command is NDRangeKernelCommand if we capture non kernel nodes
-        SetKernelName(reinterpret_cast<amd::NDRangeKernelCommand*>(command)->kernel().name());
-        command->release();
-      }
+      // Enqueue command to capture GPU Packet. The packet is not submitted to the device.
+      // The packet is stored in gpuPacket_ and submitted during graph launch.
+      command->submit(*(command->queue())->vdev());
+      // Need to ensure if the command is NDRangeKernelCommand if we capture non kernel nodes
+      SetKernelName(reinterpret_cast<amd::NDRangeKernelCommand*>(command)->kernel().name());
+      command->release();
     }
+  }
 
   std::string GetLabel(hipGraphDebugDotFlags flag) {
     hipFunction_t func = getFunc(kernelParams_, ihipGetDevice());

hipamd/src/hip_internal.hpp

@@ -487,7 +487,7 @@ public:
     void setFlags(unsigned int flags) { flags_ = flags; }
     void Reset();
 
-    hip::Stream* NullStream();
+    hip::Stream* NullStream(bool wait = true);
     Stream* GetNullStream() const {return null_stream_;};
 
     void SetActiveStatus() {
@@ -572,7 +572,7 @@ public:
   /// Get default stream associated with the ROCclr context
   extern hip::Stream* getNullStream(amd::Context&);
   /// Get default stream of the thread
-  extern hip::Stream* getNullStream();
+  extern hip::Stream* getNullStream(bool wait = true);
   /// Get device ID associated with the ROCclr context
   int getDeviceID(amd::Context& ctx);
   /// Check if stream is valid