AI/rocm-systems
2
0
Förgrening 0
Kod Ärenden Pull-förfrågningar Actions Packages Projekt Släpp Wiki Aktiviteter

SWDEV-468424 - Refactor kernel arg

Bläddra i källkod 
For refactoring of childGraph to have its own graphExec,
kernelArgs needs to be separated from the graphExec object.
All the childNodes part of graph should share same kernelArg pool.
Otherwise we endup creating multiple device kernel arg memory chucks
for single graphExec.

Change-Id: I4029a46ebc1fa112d87df64ab1fecbf288fabe5e


[ROCm/clr commit: 35079e834e]
This commit is contained in:
Anusha GodavarthySurya
2024-07-04 08:49:35 +00:00
committad av Anusha Godavarthy Surya
förälder 2e7581a69a
incheckning b6d82323e9
3 ändrade filer med 123 tillägg och 91 borttagningar
Download Patch File Download Diff File Expand all files Collapse all files
projects/clr/hipamd/src/hip_graph.cpp
+3
Visa fil
@@ -1320,6 +1320,9 @@ hipError_t ihipGraphInstantiate(hip::GraphExec** pGraphExec, hip::Graph* graph,
LogPrintfInfo("[hipGraph] graph dump:%s", filename.c_str());
}
}
if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) {
(*pGraphExec)->SetKernelArgManager(new hip::GraphKernelArgManager());
}
return (*pGraphExec)->Init();
} else {
return hipErrorOutOfMemory;
projects/clr/hipamd/src/hip_graph_internal.cpp
+77 -56
Visa fil
@@ -392,8 +392,8 @@ hipError_t AllocKernelArgForGraphNode(std::vector<hip::Node>& topoOrder,
// From the kernel pool allocate the kern arg size required for the current kernel node.
address kernArgOffset = nullptr;
if (kernelNode->GetKernargSegmentByteSize()) {
kernArgOffset = graphExec->allocKernArg(kernelNode->GetKernargSegmentByteSize(),
kernelNode->GetKernargSegmentAlignment());
kernArgOffset = graphExec->kernArgManager_->AllocKernArg(
kernelNode->GetKernargSegmentByteSize(), kernelNode->GetKernargSegmentAlignment());
if (kernArgOffset == nullptr) {
return hipErrorMemoryAllocation;
}
@@ -417,30 +417,6 @@ hipError_t AllocKernelArgForGraphNode(std::vector<hip::Node>& topoOrder,
return status;
}
// ================================================================================================
hipError_t GraphExec::AllocGraphKernargPool(size_t pool_size) {
hipError_t status = hipSuccess;
assert(pool_size > 0);
address graph_kernarg_base;
auto device = g_devices[ihipGetDevice()]->devices()[0];
if (device->info().largeBar_) {
graph_kernarg_base =
reinterpret_cast<address>(device->deviceLocalAlloc(pool_size));
device_kernarg_pool_ = true;
} else {
graph_kernarg_base = reinterpret_cast<address>(
device->hostAlloc(pool_size, 0,
amd::Device::MemorySegment::kKernArg));
}
if (graph_kernarg_base == nullptr) {
return hipErrorMemoryAllocation;
}
kernarg_graph_.push_back(KernelArgPoolGraph(graph_kernarg_base, pool_size));
return status;
}
// ================================================================================================
hipError_t GraphExec::CaptureAQLPackets() {
hipError_t status = hipSuccess;
@@ -449,33 +425,16 @@ hipError_t GraphExec::CaptureAQLPackets() {
GetKernelArgSizeForGraph(parallelLists_, kernArgSizeForGraph);
auto device = g_devices[ihipGetDevice()]->devices()[0];
// Add a larger initial pool to accomodate for any updates to kernel args
status = AllocGraphKernargPool(kernArgSizeForGraph + kKernArgChunkSize);
if (status != hipSuccess) {
return status;
bool bStatus = kernArgManager_->AllocGraphKernargPool(kernArgSizeForGraph + kKernArgChunkSize);
if (bStatus != true) {
return hipErrorMemoryAllocation;
}
status = AllocKernelArgForGraphNode(topoOrder_, capture_stream_, this);
if (status != hipSuccess) {
return status;
}
if (device_kernarg_pool_) {
auto kernArgImpl = device->settings().kernel_arg_impl_;
if (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP) {
*device->info().hdpMemFlushCntl = 1u;
auto kSentinel = *reinterpret_cast<volatile int*>(device->info().hdpMemFlushCntl);
} else if (kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback &&
kernarg_graph_.back().kernarg_pool_addr_ != 0) {
address dev_ptr = kernarg_graph_.back().kernarg_pool_addr_ +
kernarg_graph_.back().kernarg_pool_size_;
auto kSentinel = *reinterpret_cast<volatile unsigned char*>(dev_ptr - 1);
_mm_sfence();
*(dev_ptr - 1) = kSentinel;
_mm_mfence();
kSentinel = *reinterpret_cast<volatile unsigned char*>(dev_ptr - 1);
}
}
kernArgManager_->ReadBackOrFlush();
}
return status;
}
@@ -486,16 +445,11 @@ hipError_t GraphExec::UpdateAQLPacket(hip::GraphKernelNode* node) {
if (parallelLists_.size() == 1) {
size_t pool_new_usage = 0;
address kernArgOffset = nullptr;
kernArgOffset = allocKernArg(node->GetKerArgSize(),
node->GetKernargSegmentAlignment());
if (kernArgOffset == nullptr ) {
// Allocate new pool for kernarg and get the offset
status = AllocGraphKernargPool(kKernArgChunkSize);
kernArgOffset = allocKernArg(node->GetKerArgSize(),
node->GetKernargSegmentAlignment());
kernArgOffset =
kernArgManager_->AllocKernArg(node->GetKerArgSize(), node->GetKernargSegmentAlignment());
if (kernArgOffset == nullptr) {
return hipErrorMemoryAllocation;
}
node->CaptureAndFormPacket(capture_stream_, kernArgOffset);
}
return hipSuccess;
@@ -689,6 +643,7 @@ hipError_t GraphExec::Run(hipStream_t stream) {
ResetQueueIndex();
return status;
}
void ReleaseGraphExec(int deviceId) {
// Release all graph exec objects destroyed by user.
amd::ScopedLock lock(GraphExecStatusLock_);
@@ -705,6 +660,7 @@ void ReleaseGraphExec(int deviceId) {
}
}
}
void ReleaseGraphExec(hip::Stream* stream) {
amd::ScopedLock lock(GraphExecStatusLock_);
for (auto itr = GraphExecStatus_.begin(); itr != GraphExecStatus_.end();) {
@@ -721,4 +677,69 @@ void ReleaseGraphExec(hip::Stream* stream) {
}
}
}
// ================================================================================================
bool GraphKernelArgManager::AllocGraphKernargPool(size_t pool_size) {
bool bStatus = true;
assert(pool_size > 0);
address graph_kernarg_base;
auto device = g_devices[ihipGetDevice()]->devices()[0];
if (device->info().largeBar_) {
graph_kernarg_base = reinterpret_cast<address>(device->deviceLocalAlloc(pool_size));
device_kernarg_pool_ = true;
} else {
graph_kernarg_base = reinterpret_cast<address>(
device->hostAlloc(pool_size, 0, amd::Device::MemorySegment::kKernArg));
}
if (graph_kernarg_base == nullptr) {
return false;
}
kernarg_graph_.push_back(KernelArgPoolGraph(graph_kernarg_base, pool_size));
return true;
}
address GraphKernelArgManager::AllocKernArg(size_t size, size_t alignment) {
assert(alignment != 0);
address result = nullptr;
result = amd::alignUp(
kernarg_graph_.back().kernarg_pool_addr_ + kernarg_graph_.back().kernarg_pool_offset_,
alignment);
const size_t pool_new_usage = (result + size) - kernarg_graph_.back().kernarg_pool_addr_;
if (pool_new_usage <= kernarg_graph_.back().kernarg_pool_size_) {
kernarg_graph_.back().kernarg_pool_offset_ = pool_new_usage;
} else {
// If current chunck is full allocate new chunck with same size as current
bool bStatus = AllocGraphKernargPool(kernarg_graph_.back().kernarg_pool_size_);
if (bStatus == false) {
return nullptr;
} else {
// Allocte kernel arg memory from new chunck
return AllocKernArg(size, alignment);
}
}
return result;
}
void GraphKernelArgManager::ReadBackOrFlush() {
if (device_kernarg_pool_) {
auto device = g_devices[ihipGetDevice()]->devices()[0];
auto kernArgImpl = device->settings().kernel_arg_impl_;
if (kernArgImpl == KernelArgImpl::DeviceKernelArgsHDP) {
*device->info().hdpMemFlushCntl = 1u;
auto kSentinel = *reinterpret_cast<volatile int*>(device->info().hdpMemFlushCntl);
} else if (kernArgImpl == KernelArgImpl::DeviceKernelArgsReadback &&
kernarg_graph_.back().kernarg_pool_addr_ != 0) {
address dev_ptr =
kernarg_graph_.back().kernarg_pool_addr_ + kernarg_graph_.back().kernarg_pool_size_;
auto kSentinel = *reinterpret_cast<volatile unsigned char*>(dev_ptr - 1);
_mm_sfence();
*(dev_ptr - 1) = kSentinel;
_mm_mfence();
kSentinel = *reinterpret_cast<volatile unsigned char*>(dev_ptr - 1);
}
}
}
} // namespace hip
projects/clr/hipamd/src/hip_graph_internal.hpp
+43 -35
Visa fil
@@ -552,18 +552,42 @@ struct Graph {
}
};
struct GraphKernelNode;
struct GraphExec : public amd::ReferenceCountedObject {
struct GraphKernelArgManager : public amd::ReferenceCountedObject {
public:
GraphKernelArgManager() : ReferenceCountedObject() {}
~GraphKernelArgManager() {
//! Release the kernel arg pools
auto device = g_devices[ihipGetDevice()]->devices()[0];
for (auto& element : kernarg_graph_) {
device->hostFree(element.kernarg_pool_addr_, element.kernarg_pool_size_);
}
kernarg_graph_.clear();
}
// Allocate kernel arg pool for the given size.
bool AllocGraphKernargPool(size_t pool_size);
// Allocate kernel args from current chunck for given size and alignment.
// If kernel arg pool is full allocate new chunck and alloc kern args from new pool.
address AllocKernArg(size_t size, size_t alignment);
// Do HDP flush/When HDP flush register is invalid fallback to Readback
void ReadBackOrFlush();
private:
struct KernelArgPoolGraph {
KernelArgPoolGraph(address base_addr, size_t size)
: kernarg_pool_addr_(base_addr),
kernarg_pool_size_(size),
kernarg_pool_offset_(0)
{}
: kernarg_pool_addr_(base_addr), kernarg_pool_size_(size), kernarg_pool_offset_(0) {}
address kernarg_pool_addr_; //! Base address of the kernel arg pool
size_t kernarg_pool_size_; //! Size of the pool
size_t kernarg_pool_offset_; //! Current offset in the kernel arg alloc
};
bool device_kernarg_pool_ = false; //! Indicate if kernel pool in device mem
std::vector<KernelArgPoolGraph> kernarg_graph_; //! Vector of allocated kernarg pool
using KernelArgImpl = device::Settings::KernelArgImpl;
};
struct GraphExec : public amd::ReferenceCountedObject {
std::vector<std::vector<Node>> parallelLists_;
//! Topological order of the graph doesn't include nodes embedded as part of the child graph
std::vector<Node> topoOrder_;
@@ -578,11 +602,7 @@ struct GraphExec : public amd::ReferenceCountedObject {
static amd::Monitor graphExecSetLock_;
uint64_t flags_ = 0;
bool repeatLaunch_ = false;
bool device_kernarg_pool_ = false; //! Indicate if kernel pool in device mem
std::vector<KernelArgPoolGraph> kernarg_graph_; //! Vector of allocated kernarg pool
uint32_t kernarg_graph_cur_offset_ = 0; //! Current offset in kernarg pool
GraphKernelArgManager* kernArgManager_ = nullptr; //!< Kernel Arg manager for graph.
int instantiateDeviceId_ = -1;
bool hasHiddenHeap_ = false; //!< Hidden heap indicator for Kernel node
@@ -609,17 +629,12 @@ struct GraphExec : public amd::ReferenceCountedObject {
hip::Stream::Destroy(stream);
}
}
//! Release the kernel arg pools
if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) {
auto device = g_devices[ihipGetDevice()]->devices()[0];
for (auto& element : kernarg_graph_) {
device->hostFree(element.kernarg_pool_addr_, element.kernarg_pool_size_);
}
}
amd::ScopedLock lock(graphExecSetLock_);
graphExecSet_.erase(this);
delete clonedGraph_;
if (DEBUG_CLR_GRAPH_PACKET_CAPTURE) {
kernArgManager_->release();
}
}
Node GetClonedNode(Node node) {
@@ -636,24 +651,8 @@ struct GraphExec : public amd::ReferenceCountedObject {
//! Graph has nodes that require hidden heap.
void SetHiddenHeap() { hasHiddenHeap_ = true; }
address allocKernArg(size_t size, size_t alignment) {
assert(alignment != 0);
address result = nullptr;
result = amd::alignUp(kernarg_graph_.back().kernarg_pool_addr_ +
kernarg_graph_.back().kernarg_pool_offset_,
alignment);
const size_t pool_new_usage = (result + size) - kernarg_graph_.back().kernarg_pool_addr_;
if (pool_new_usage <= kernarg_graph_.back().kernarg_pool_size_) {
kernarg_graph_.back().kernarg_pool_offset_ = pool_new_usage;
} else {
return nullptr;
}
return result;
}
//! Check executable graphs validity
static bool isGraphExecValid(GraphExec* pGraphExec);
hipError_t AllocGraphKernargPool(size_t pool_size);
std::vector<Node>& GetNodes() { return topoOrder_; }
hip::Stream* GetAvailableStreams() {
@@ -671,7 +670,16 @@ struct GraphExec : public amd::ReferenceCountedObject {
// Capture GPU Packets from graph commands
hipError_t CaptureAQLPackets();
hipError_t UpdateAQLPacket(hip::GraphKernelNode* node);
using KernelArgImpl = device::Settings::KernelArgImpl;
// Kenrel arg manger is for the entire graph.
// Child graph also shares the same kernel arg manager object. some apps have 100's of
// child graph nodes and each child graph has only one node.
void SetKernelArgManager(GraphKernelArgManager* kernArgManager) {
kernArgManager_ = kernArgManager;
kernArgManager_->retain();
}
GraphKernelArgManager* GetKernelArgManager() {
return kernArgManager_;
}
};
struct ChildGraphNode : public GraphNode {