diff --git a/hipamd/src/hip_graph.cpp b/hipamd/src/hip_graph.cpp index 754d3ecb01..a737700ca8 100644 --- a/hipamd/src/hip_graph.cpp +++ b/hipamd/src/hip_graph.cpp @@ -1651,7 +1651,15 @@ hipError_t hipGraphExecMemsetNodeSetParams(hipGraphExec_t hGraphExec, hipGraphNo if (clonedNode == nullptr) { HIP_RETURN(hipErrorInvalidValue); } - HIP_RETURN(reinterpret_cast(clonedNode)->SetParams(pNodeParams, true)); + hipError_t status = reinterpret_cast(clonedNode)->SetParams(pNodeParams, true); + if(status != hipSuccess) { + HIP_RETURN(status); + } + if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) { + status = reinterpret_cast(hGraphExec) + ->UpdateAQLPacket(reinterpret_cast(clonedNode)); + } + HIP_RETURN(status); } hipError_t hipGraphAddDependencies(hipGraph_t graph, const hipGraphNode_t* from, @@ -1773,7 +1781,7 @@ hipError_t hipGraphExecChildGraphNodeSetParams(hipGraphExec_t hGraphExec, hipGra std::vector childGraphNodes; reinterpret_cast(clonedNode)->TopologicalOrder(childGraphNodes); for (std::vector::size_type i = 0; i != childGraphNodes.size(); i++) { - if (childGraphNodes[i]->GetType() == hipGraphNodeTypeKernel) { + if (childGraphNodes[i]->GraphCaptureEnabled()) { status = reinterpret_cast(hGraphExec) ->UpdateAQLPacket(reinterpret_cast(childGraphNodes[i])); if (status != hipSuccess) { @@ -2478,8 +2486,7 @@ hipError_t hipGraphExecUpdate(hipGraphExec_t hGraphExec, hipGraph_t hGraph, *updateResult_out = hipGraphExecUpdateErrorNotSupported; } HIP_RETURN(hipErrorGraphExecUpdateFailure); - } else if (DEBUG_CLR_GRAPH_PACKET_CAPTURE && - oldGraphExecNodes[i]->GetType() == hipGraphNodeTypeKernel) { + } else if (DEBUG_CLR_GRAPH_PACKET_CAPTURE && newGraphNodes[i]->GraphCaptureEnabled()) { status = reinterpret_cast(hGraphExec) ->UpdateAQLPacket(reinterpret_cast(oldGraphExecNodes[i])); diff --git a/hipamd/src/hip_graph_internal.cpp b/hipamd/src/hip_graph_internal.cpp index 77ce47d490..e0b5e90a59 100644 --- a/hipamd/src/hip_graph_internal.cpp +++ b/hipamd/src/hip_graph_internal.cpp @@ -392,17 +392,7 @@ hipError_t AllocKernelArgForGraphNode(std::vector& topoOrder, } } if (node->GraphCaptureEnabled()) { - // From the kernel pool allocate the kern arg size required for the current node. - address kernArgOffset = nullptr; - if (node->GetKernargSegmentByteSize()) { - kernArgOffset = graphExec->kernArgManager_->AllocKernArg( - node->GetKernargSegmentByteSize(), node->GetKernargSegmentAlignment()); - if (kernArgOffset == nullptr) { - return hipErrorMemoryAllocation; - } - } - // Form GPU packet capture for the kernel node. - node->CaptureAndFormPacket(capture_stream, kernArgOffset); + node->CaptureAndFormPacket(capture_stream, graphExec->GetKernelArgManager()); } else if (node->GetType() == hipGraphNodeTypeGraph) { auto childNode = reinterpret_cast(node); auto& childParallelLists = childNode->GetParallelLists(); @@ -443,17 +433,10 @@ hipError_t GraphExec::CaptureAQLPackets() { } // ================================================================================================ -hipError_t GraphExec::UpdateAQLPacket(hip::GraphKernelNode* node) { +hipError_t GraphExec::UpdateAQLPacket(hip::GraphNode* node) { hipError_t status = hipSuccess; if (parallelLists_.size() == 1) { - size_t pool_new_usage = 0; - address kernArgOffset = nullptr; - kernArgOffset = - kernArgManager_->AllocKernArg(node->GetKerArgSize(), node->GetKernargSegmentAlignment()); - if (kernArgOffset == nullptr) { - return hipErrorMemoryAllocation; - } - node->CaptureAndFormPacket(capture_stream_, kernArgOffset); + node->CaptureAndFormPacket(capture_stream_, kernArgManager_); } return hipSuccess; } @@ -556,9 +539,10 @@ hipError_t EnqueueGraphWithSingleList(std::vector& topoOrder, hip::St for (int i = 0; i < topoOrder.size(); i++) { if (topoOrder[i]->GraphCaptureEnabled()) { if (topoOrder[i]->GetEnabled()) { - hip_stream->vdev()->dispatchAqlPacket(topoOrder[i]->GetAqlPacket(), - topoOrder[i]->GetKernelName(), - accumulate); + std::vector& gpuPackets = topoOrder[i]->GetAqlPackets(); + for (auto& packet : gpuPackets) { + hip_stream->vdev()->dispatchAqlPacket(packet, topoOrder[i]->GetKernelName(), accumulate); + } } } else { topoOrder[i]->SetStream(hip_stream, graphExec); diff --git a/hipamd/src/hip_graph_internal.hpp b/hipamd/src/hip_graph_internal.hpp index 88be5edbbb..d706d9a43f 100644 --- a/hipamd/src/hip_graph_internal.hpp +++ b/hipamd/src/hip_graph_internal.hpp @@ -166,6 +166,41 @@ struct hipGraphNodeDOTAttribute { } }; +class GraphKernelArgManager : public amd::ReferenceCountedObject, public amd::GraphKernelArgManager { + public: + GraphKernelArgManager() : amd::ReferenceCountedObject() {} + ~GraphKernelArgManager() { + //! Release the kernel arg pools + auto device = g_devices[ihipGetDevice()]->devices()[0]; + for (auto& element : kernarg_graph_) { + device->hostFree(element.kernarg_pool_addr_, element.kernarg_pool_size_); + } + kernarg_graph_.clear(); + } + + // Allocate kernel arg pool for the given size. + bool AllocGraphKernargPool(size_t pool_size); + + // Allocate kernel args from current chunck for given size and alignment. + // If kernel arg pool is full allocate new chunck and alloc kern args from new pool. + address AllocKernArg(size_t size, size_t alignment) override; + + // Do HDP flush/When HDP flush register is invalid fallback to Readback + void ReadBackOrFlush(); + + private: + struct KernelArgPoolGraph { + KernelArgPoolGraph(address base_addr, size_t size) + : kernarg_pool_addr_(base_addr), kernarg_pool_size_(size), kernarg_pool_offset_(0) {} + address kernarg_pool_addr_; //! Base address of the kernel arg pool + size_t kernarg_pool_size_; //! Size of the pool + size_t kernarg_pool_offset_; //! Current offset in the kernel arg alloc + }; + bool device_kernarg_pool_ = false; //! Indicate if kernel pool in device mem + std::vector kernarg_graph_; //! Vector of allocated kernarg pool + using KernelArgImpl = device::Settings::KernelArgImpl; +}; + struct GraphNode : public hipGraphNodeDOTAttribute { protected: hip::Stream* stream_ = nullptr; @@ -184,7 +219,7 @@ struct GraphNode : public hipGraphNodeDOTAttribute { static std::unordered_set nodeSet_; static amd::Monitor nodeSetLock_; unsigned int isEnabled_; - uint8_t gpuPacket_[64]; //!< GPU Packet to enqueue during graph launch + std::vector gpuPackets_; //!< GPU Packet to enqueue during graph launch std::string capturedKernelName_; size_t alignedKernArgSize_ = 256; //!< Aligned size required for kernel args size_t kernargSegmentByteSize_ = 256; //!< Kernel arg segment byte size @@ -224,6 +259,9 @@ struct GraphNode : public hipGraphNodeDOTAttribute { for (auto node : dependencies_) { node->RemoveEdge(this); } + for (auto packet : gpuPackets_) { + delete packet; + } amd::ScopedLock lock(nodeSetLock_); nodeSet_.erase(this); } @@ -237,16 +275,16 @@ struct GraphNode : public hipGraphNodeDOTAttribute { return true; } // Return gpu packet address to update with actual packet under capture. - uint8_t* GetAqlPacket() { return gpuPacket_; } + std::vector& GetAqlPackets() { return gpuPackets_; } void SetKernelName(const std::string& kernelName) { capturedKernelName_ = kernelName; } const std::string& GetKernelName() const { return capturedKernelName_; } size_t GetKerArgSize() const { return alignedKernArgSize_; } size_t GetKernargSegmentByteSize() const { return kernargSegmentByteSize_; } size_t GetKernargSegmentAlignment() const { return kernargSegmentAlignment_; } - void CaptureAndFormPacket(hip::Stream* capture_stream, address kernArgOffset) { + void CaptureAndFormPacket(hip::Stream* capture_stream, GraphKernelArgManager* kernArgMgr) { hipError_t status = CreateCommand(capture_stream); for (auto& command : commands_) { - command->setCapturingState(true, GetAqlPacket(), kernArgOffset, &capturedKernelName_); + command->setCapturingState(true, &gpuPackets_, kernArgMgr, &capturedKernelName_); // Enqueue command to capture GPU Packet. The packet is not submitted to the device. // The packet is stored in gpuPacket_ and submitted during graph launch. command->submit(*(command->queue())->vdev()); @@ -404,6 +442,9 @@ struct GraphNode : public hipGraphNodeDOTAttribute { case hipGraphNodeTypeKernel: isGraphCapture = true; break; + case hipGraphNodeTypeMemset: + isGraphCapture = true; + break; default: break; } @@ -582,40 +623,6 @@ struct Graph { } }; struct GraphKernelNode; -struct GraphKernelArgManager : public amd::ReferenceCountedObject { - public: - GraphKernelArgManager() : ReferenceCountedObject() {} - ~GraphKernelArgManager() { - //! Release the kernel arg pools - auto device = g_devices[ihipGetDevice()]->devices()[0]; - for (auto& element : kernarg_graph_) { - device->hostFree(element.kernarg_pool_addr_, element.kernarg_pool_size_); - } - kernarg_graph_.clear(); - } - - // Allocate kernel arg pool for the given size. - bool AllocGraphKernargPool(size_t pool_size); - - // Allocate kernel args from current chunck for given size and alignment. - // If kernel arg pool is full allocate new chunck and alloc kern args from new pool. - address AllocKernArg(size_t size, size_t alignment); - - // Do HDP flush/When HDP flush register is invalid fallback to Readback - void ReadBackOrFlush(); - - private: - struct KernelArgPoolGraph { - KernelArgPoolGraph(address base_addr, size_t size) - : kernarg_pool_addr_(base_addr), kernarg_pool_size_(size), kernarg_pool_offset_(0) {} - address kernarg_pool_addr_; //! Base address of the kernel arg pool - size_t kernarg_pool_size_; //! Size of the pool - size_t kernarg_pool_offset_; //! Current offset in the kernel arg alloc - }; - bool device_kernarg_pool_ = false; //! Indicate if kernel pool in device mem - std::vector kernarg_graph_; //! Vector of allocated kernarg pool - using KernelArgImpl = device::Settings::KernelArgImpl; -}; struct GraphExec : public amd::ReferenceCountedObject { std::vector> parallelLists_; @@ -699,7 +706,7 @@ struct GraphExec : public amd::ReferenceCountedObject { hipError_t Run(hipStream_t stream); // Capture GPU Packets from graph commands hipError_t CaptureAQLPackets(); - hipError_t UpdateAQLPacket(hip::GraphKernelNode* node); + hipError_t UpdateAQLPacket(hip::GraphNode* node); // Kenrel arg manger is for the entire graph. // Child graph also shares the same kernel arg manager object. some apps have 100's of // child graph nodes and each child graph has only one node. diff --git a/rocclr/device/rocm/rocvirtual.cpp b/rocclr/device/rocm/rocvirtual.cpp index 111329e6b0..1b896d98f6 100644 --- a/rocclr/device/rocm/rocvirtual.cpp +++ b/rocclr/device/rocm/rocvirtual.cpp @@ -3242,7 +3242,8 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, if (!kernel.parameters().deviceKernelArgs() || gpuKernel.isInternalKernel()) { // Allocate buffer to hold kernel arguments if (isGraphCapture) { - argBuffer = currCmd_->getKernArgOffset(); + argBuffer = currCmd_->getKernArgOffset(gpuKernel.KernargSegmentByteSize(), + gpuKernel.KernargSegmentAlignment()); currCmd_->SetKernelName(gpuKernel.name()); } else { diff --git a/rocclr/platform/command.hpp b/rocclr/platform/command.hpp index af70bb28ab..101d1f67d3 100644 --- a/rocclr/platform/command.hpp +++ b/rocclr/platform/command.hpp @@ -239,6 +239,12 @@ union CopyMetadata { copyEnginePreference_(copyEnginePreference) {} }; +// Interface to callback to allocate kernel args from the graph kernel arg pool. +class GraphKernelArgManager { + public: + virtual address AllocKernArg(size_t size, size_t alignment) = 0; +}; + /*! \brief An operation that is submitted to a command queue. * * %Command is the abstract base type of all OpenCL operations @@ -257,7 +263,8 @@ class Command : public Event { const Event* waitingEvent_; //!< Waiting event associated with the marker bool capturing_ = false; //!< Flag to enable/disable graph gpu packet capture - uint8_t* gpuPacket_ = nullptr; //!< GPU packet to capture, when graph capturing is enabled + std::vector* gpuPackets_; //!< GPU packets captured when graph capturing is enabled + GraphKernelArgManager* graphKernArgMgr_ = nullptr; //!< KernelMgr for graph address kernArgOffset_ = nullptr; //!< KernelArg buffer to used when graph capturing is enabled std::string* capturedKernelName_ = nullptr; //!< Kenrnel under capture protected: @@ -300,11 +307,12 @@ class Command : public Event { bool getCapturingState() const { return capturing_; } //! Sets AQL capture state, aql packet to capture and where to copy kernArgs - void setCapturingState(bool state, uint8_t* packet, address kernArgOffset, + void setCapturingState(bool state, std::vector* packet, + amd::GraphKernelArgManager* graphKernArgMgr, std::string* capturedKernelName) { capturing_ = state; - gpuPacket_ = packet; - kernArgOffset_ = kernArgOffset; + gpuPackets_ = packet; + graphKernArgMgr_ = graphKernArgMgr; capturedKernelName_ = capturedKernelName; } @@ -315,11 +323,16 @@ class Command : public Event { } } - //! returns the graph executable object command belongs to. - const uint8_t* getAqlPacket() const { return gpuPacket_; } + //! Returns the graph executable object command belongs to. + const uint8_t* getAqlPacket() const { + uint8_t* packet = new uint8_t[64]; + gpuPackets_->push_back(packet); + return packet; + } - //! returns the graph executable object command belongs to. - const address getKernArgOffset() const { return kernArgOffset_; } + address getKernArgOffset(int size, int alignment) { + return graphKernArgMgr_->AllocKernArg(size, alignment); + } //! Overload new/delete for fast commands allocation/destruction void* operator new(size_t size);