AI/rocm-systems
2
0
Förgrening 0
Kod Ärenden Pull-förfrågningar Actions Packages Projekt Släpp Wiki Aktiviteter

SWDEV-422207 - Capture AQL Packets for graph Kernel nodes during graph Inst. And enqueue AQL packet during launch

Bläddra i källkod 
Change-Id: I1e5f7f9e2a70bd500d190193cb6ba0867f5a63e7
This commit is contained in:
Anusha GodavarthySurya
2023-07-19 10:10:37 +00:00
committad av Anusha Godavarthy Surya
förälder 3eb46ae588
incheckning e63c280d4d
10 ändrade filer med 237 tillägg och 174 borttagningar
Download Patch File Download Diff File Expand all files Collapse all files
hipamd/src/hip_graph.cpp
+4
Visa fil
@@ -1177,6 +1177,10 @@ hipError_t hipGraphInstantiate(hipGraphExec_t* pGraphExec, hipGraph_t graph,
hip::GraphExec* ge;
hipError_t status = ihipGraphInstantiate(&ge, reinterpret_cast<hip::Graph*>(graph));
*pGraphExec = reinterpret_cast<hipGraphExec_t>(ge);
if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) {
// For graph nodes capture AQL packets to dispatch them directly during graph launch.
status = ge->CaptureAQLPackets();
}
HIP_RETURN(status);
}
hipamd/src/hip_graph_internal.cpp
+77 -14
Visa fil
@@ -480,7 +480,7 @@ hipError_t GraphExec::CreateStreams(uint32_t num_streams) {
}
hipError_t GraphExec::Init() {
hipError_t status;
hipError_t status = hipSuccess;
size_t min_num_streams = 1;
for (auto& node : topoOrder_) {
@@ -493,11 +493,63 @@ hipError_t GraphExec::Init() {
return status;
}
hipError_t GraphExec::CaptureAQLPackets() {
hipError_t status = hipSuccess;
size_t KernArgSizeForGraph = 0;
bool GraphHasOnlyKerns = true;
// GPU packet capture is enabled for kernel nodes. Calculate the kernel arg size required for all
// graph kernel nodes to allocate
for (const auto& list : parallelLists_) {
hip::Stream* stream = GetAvailableStreams();
for (auto& node : list) {
node->SetStream(stream, this);
if (node->GetType() == hipGraphNodeTypeKernel) {
KernArgSizeForGraph += reinterpret_cast<hip::GraphKernelNode*>(node)->GetKerArgSize();
} else {
GraphHasOnlyKerns = false;
}
}
}
auto device = g_devices[ihipGetDevice()]->devices()[0];
const auto& info = device->info();
// Enable allocating kerns on device memory if graph as only kernels. memcpy nodes require hdp
// flush. ToDo: Work on enabling device kern args later for all type of nodes for large bar
if (GraphHasOnlyKerns == true && info.largeBar_) {
kernarg_pool_graph_ = reinterpret_cast<address>(device->deviceLocalAlloc(KernArgSizeForGraph));
device_kernarg_pool_ = true;
} else {
kernarg_pool_graph_ = reinterpret_cast<address>(
device->hostAlloc(KernArgSizeForGraph, 0, amd::Device::MemorySegment::kKernArg));
}
if (kernarg_pool_graph_ == nullptr) {
return hipErrorMemoryAllocation;
}
kernarg_pool_size_graph_ = KernArgSizeForGraph;
for (auto& node : topoOrder_) {
if (node->GetType() == hipGraphNodeTypeKernel) {
auto kernelnode = reinterpret_cast<hip::GraphKernelNode*>(node);
status = node->CreateCommand(node->GetQueue());
// From the kernel pool allocate the kern arg size required for the current kernel node.
address kernArgOffset = allocKernArg(kernelnode->GetKernargSegmentByteSize(),
kernelnode->GetKernargSegmentAlignment());
if (kernArgOffset == nullptr) {
return hipErrorMemoryAllocation;
}
// Enable GPU packet capture for the kernel node.
kernelnode->EnableCapturing(kernArgOffset);
}
}
return status;
}
hipError_t FillCommands(std::vector<std::vector<Node>>& parallelLists,
std::unordered_map<Node, std::vector<Node>>& nodeWaitLists,
std::vector<Node>& topoOrder, Graph* clonedGraph,
amd::Command*& graphStart, amd::Command*& graphEnd, hip::Stream* stream) {
hipError_t status;
hipError_t status = hipSuccess;
for (auto& node : topoOrder) {
// TODO: clone commands from next launch
status = node->CreateCommand(node->GetQueue());
@@ -578,7 +630,7 @@ void UpdateStream(std::vector<std::vector<Node>>& parallelLists, hip::Stream* st
}
hipError_t GraphExec::Run(hipStream_t stream) {
hipError_t status;
hipError_t status = hipSuccess;
if (hip::getStream(stream) == nullptr) {
return hipErrorInvalidResourceHandle;
@@ -603,19 +655,30 @@ hipError_t GraphExec::Run(hipStream_t stream) {
repeatLaunch_ = true;
}
if (parallelLists_.size() == 1) {
if (device_kernarg_pool_) {
// If kernelArgs are in device memory flush the HDP.
amd::Command* startCommand = nullptr;
startCommand = new amd::Marker(*hip_stream, false);
startCommand->enqueue();
startCommand->release();
}
for (int i = 0; i < topoOrder_.size(); i++) {
topoOrder_[i]->SetStream(hip_stream, this);
status = topoOrder_[i]->CreateCommand(topoOrder_[i]->GetQueue());
if (DEBUG_CLR_GRAPH_ENABLE_BUFFERING) {
// Enable buffering for graph with single branch
// Peep through the next node. If current and next node are kernel then enable AQL
// buffering
if (((i + 1) != topoOrder_.size()) && topoOrder_[i]->GetType() == hipGraphNodeTypeKernel &&
topoOrder_[i + 1]->GetType() == hipGraphNodeTypeKernel) {
topoOrder_[i]->EnableBuffering();
}
if (DEBUG_CLR_GRAPH_PACKET_CAPTURE && topoOrder_[i]->GetType() == hipGraphNodeTypeKernel) {
hip_stream->vdev()->dispatchAqlPacket(topoOrder_[i]->GetAqlPacket());
} else {
topoOrder_[i]->SetStream(hip_stream, this);
status = topoOrder_[i]->CreateCommand(topoOrder_[i]->GetQueue());
topoOrder_[i]->EnqueueCommands(stream);
}
topoOrder_[i]->EnqueueCommands(stream);
}
if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) {
amd::Command* endCommand = nullptr;
endCommand = new amd::Marker(*hip_stream, false);
// Since the end command is for graph completion tracking,
// it may not need release scopes
endCommand->setEventScope(amd::Device::kCacheStateIgnore);
endCommand->enqueue();
endCommand->release();
}
} else {
UpdateStream(parallelLists_, hip_stream, this);
hipamd/src/hip_graph_internal.hpp
+72 -30
Visa fil
@@ -182,6 +182,7 @@ struct GraphNode : public hipGraphNodeDOTAttribute {
static std::unordered_set<GraphNode*> nodeSet_;
static amd::Monitor nodeSetLock_;
unsigned int isEnabled_;
uint8_t gpuPacket_[64]; //!< GPU Packet to enqueue during graph launch
public:
GraphNode(hipGraphNodeType type, std::string style = "", std::string shape = "",
@@ -229,7 +230,8 @@ struct GraphNode : public hipGraphNodeDOTAttribute {
}
return true;
}
// Return gpu packet address to update with actual packet under capture.
uint8_t* GetAqlPacket() { return gpuPacket_; }
hip::Stream* GetQueue() { return stream_; }
virtual void SetStream(hip::Stream* stream, GraphExec* ptr = nullptr) {
@@ -336,11 +338,6 @@ struct GraphNode : public hipGraphNodeDOTAttribute {
command->release();
}
}
virtual void EnableBuffering() {
for (auto& command : commands_) {
command->setBufferingState(true);
}
}
Graph* GetParentGraph() { return parentGraph_; }
virtual Graph* GetChildGraph() { return nullptr; }
void SetParentGraph(Graph* graph) { parentGraph_ = graph; }
@@ -567,7 +564,11 @@ struct GraphExec {
static amd::Monitor graphExecSetLock_;
uint64_t flags_ = 0;
bool repeatLaunch_ = false;
// Graph Kernel arg vars
bool device_kernarg_pool_ = false;
address kernarg_pool_graph_ = nullptr;
uint32_t kernarg_pool_size_graph_ = 0;
uint32_t kernarg_pool_cur_graph_offset_ = 0;
public:
GraphExec(std::vector<Node>& topoOrder, std::vector<std::vector<Node>>& lists,
std::unordered_map<Node, std::vector<Node>>& nodeWaitLists, struct Graph*& clonedGraph,
@@ -592,6 +593,11 @@ struct GraphExec {
hip::Stream::Destroy(stream);
}
}
// Release the kernel arg memory.
auto device = g_devices[ihipGetDevice()]->devices()[0];
if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) {
device->hostFree(kernarg_pool_graph_, kernarg_pool_size_graph_);
}
amd::ScopedLock lock(graphExecSetLock_);
graphExecSet_.erase(this);
delete clonedGraph_;
@@ -606,7 +612,16 @@ struct GraphExec {
}
return clonedNode;
}
address allocKernArg(size_t size, size_t alignment) {
assert(alignment != 0);
address result = nullptr;
result = amd::alignUp(kernarg_pool_graph_ + kernarg_pool_cur_graph_offset_, alignment);
const size_t pool_new_usage = (result + size) - kernarg_pool_graph_;
if (pool_new_usage <= kernarg_pool_size_graph_) {
kernarg_pool_cur_graph_offset_ = pool_new_usage;
}
return result;
}
// check executable graphs validity
static bool isGraphExecValid(GraphExec* pGraphExec);
@@ -617,6 +632,8 @@ struct GraphExec {
hipError_t Init();
hipError_t CreateStreams(uint32_t num_streams);
hipError_t Run(hipStream_t stream);
// Capture GPU Packets from graph commands
hipError_t CaptureAQLPackets();
};
struct ChildGraphNode : public GraphNode {
@@ -742,31 +759,48 @@ struct ChildGraphNode : public GraphNode {
};
class GraphKernelNode : public GraphNode {
hipKernelNodeParams kernelParams_;
unsigned int numParams_;
hipKernelNodeAttrValue kernelAttr_;
unsigned int kernelAttrInUse_;
ihipExtKernelEvents kernelEvents_;
hipKernelNodeParams kernelParams_; //!< Kernel node parameters
unsigned int numParams_; //!< No. of kernel params as part of signature
hipKernelNodeAttrValue kernelAttr_; //!< Kernel node attributes
unsigned int kernelAttrInUse_; //!< Kernel attributes in use
ihipExtKernelEvents kernelEvents_; //!< Events for Ext launch kernel
size_t alignedKernArgSize_; //!< Aligned size required for kernel args
size_t kernargSegmentByteSize_; //!< Kernel arg segment byte size
size_t kernargSegmentAlignment_; //!< Kernel arg segment alignment
public:
void PrintAttributes(std::ostream& out, hipGraphDebugDotFlags flag) {
out << "[";
out << "style";
out << "=\"";
out << style_;
(flag == hipGraphDebugDotFlagsKernelNodeParams ||
flag == hipGraphDebugDotFlagsKernelNodeAttributes) ?
out << "\n" : out << "\"";
out << "shape";
out << "=\"";
out << GetShape(flag);
out << "\"";
out << "label";
out << "=\"";
out << GetLabel(flag);
out << "\"";
out << "];";
size_t GetKerArgSize() const { return alignedKernArgSize_; }
size_t GetKernargSegmentByteSize() const { return kernargSegmentByteSize_; }
size_t GetKernargSegmentAlignment() const { return kernargSegmentAlignment_; }
void PrintAttributes(std::ostream& out, hipGraphDebugDotFlags flag) {
out << "[";
out << "style";
out << "=\"";
out << style_;
(flag == hipGraphDebugDotFlagsKernelNodeParams ||
flag == hipGraphDebugDotFlagsKernelNodeAttributes) ?
out << "\n" : out << "\"";
out << "shape";
out << "=\"";
out << GetShape(flag);
out << "\"";
out << "label";
out << "=\"";
out << GetLabel(flag);
out << "\"";
out << "];";
}
void EnableCapturing(address kernArgOffset) {
for (auto& command : commands_) {
reinterpret_cast<amd::NDRangeKernelCommand*>(command)->setCapturingState(
true, GetAqlPacket(), kernArgOffset);
// Enqueue command to capture GPU Packet. Packet is not sent to hardware queue.
command->submit(*(command->queue())->vdev());
command->release();
}
}
std::string GetLabel(hipGraphDebugDotFlags flag) {
hipFunction_t func = getFunc(kernelParams_, ihipGetDevice());
@@ -842,6 +876,14 @@ class GraphKernelNode : public GraphNode {
}
hip::DeviceFunc* function = hip::DeviceFunc::asFunction(func);
amd::Kernel* kernel = function->kernel();
if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) {
auto device = g_devices[ihipGetDevice()]->devices()[0];
device::Kernel* devKernel = const_cast<device::Kernel*>(kernel->getDeviceKernel(*device));
kernargSegmentByteSize_ = devKernel->KernargSegmentByteSize();
kernargSegmentAlignment_ = devKernel->KernargSegmentAlignment();
alignedKernArgSize_ =
amd::alignUp(devKernel->KernargSegmentByteSize(), devKernel->KernargSegmentAlignment());
}
const amd::KernelSignature& signature = kernel->signature();
numParams_ = signature.numParameters();