From 5adbed7c63cedd80a06696595957d5578970e8a2 Mon Sep 17 00:00:00 2001 From: foreman Date: Mon, 12 Aug 2019 18:36:49 -0400 Subject: [PATCH] P4 to Git Change 1982032 by gandryey@gera-win10 on 2019/08/12 18:30:49 SWDEV-199667 - [OpenCL][PAL][LC] bruteforce conformance test causes vm fault - With HWS scratch buffer has to be allocated per each windows scheduling context, because HWS can schedule CB on any pipe and different pipes can't preserve unique wave_id. Recycle PAL queues in order to keep the scratch buffer per scheduling context. The change will also allow to remove Windows/kmd limitation of the scheduling contexts per process. GPU_MAX_HW_QUEUES controls the number of unique PAL queues, default is 4 Affected files ... ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#154 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.hpp#42 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#146 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.hpp#63 edit [ROCm/clr commit: 884478d0aaa88769b5c5a474540ba92a59d79249] --- .../rocclr/runtime/device/pal/paldevice.cpp | 3 +- .../rocclr/runtime/device/pal/paldevice.hpp | 32 ++-- .../rocclr/runtime/device/pal/palvirtual.cpp | 140 +++++++++++++----- .../rocclr/runtime/device/pal/palvirtual.hpp | 18 ++- 4 files changed, 141 insertions(+), 52 deletions(-) diff --git a/projects/clr/rocclr/runtime/device/pal/paldevice.cpp b/projects/clr/rocclr/runtime/device/pal/paldevice.cpp index 32d7bb063a..3c18de4411 100644 --- a/projects/clr/rocclr/runtime/device/pal/paldevice.cpp +++ b/projects/clr/rocclr/runtime/device/pal/paldevice.cpp @@ -1155,8 +1155,7 @@ bool Device::initializeHeapResources() { heapInitComplete_ = true; - scratch_.resize( - (settings().useSingleScratch_) ? 1 : (numComputeEngines() ? numComputeEngines() : 1)); + scratch_.resize(GPU_MAX_HW_QUEUES + numExclusiveComputeEngines()); // Initialize the number of mem object for the scratch buffer for (uint s = 0; s < scratch_.size(); ++s) { diff --git a/projects/clr/rocclr/runtime/device/pal/paldevice.hpp b/projects/clr/rocclr/runtime/device/pal/paldevice.hpp index 1e662fea8b..160a0a554c 100644 --- a/projects/clr/rocclr/runtime/device/pal/paldevice.hpp +++ b/projects/clr/rocclr/runtime/device/pal/paldevice.hpp @@ -181,6 +181,14 @@ class Sampler : public device::Sampler { //! A GPU device ordinal (physical GPU device) class Device : public NullDevice { public: + struct QueueRecycleInfo : public amd::HeapObject { + int counter_; //!< Lock usage counter + Pal::EngineType engineType_; //!< Engine type + uint32_t index_; //!< HW queue index for scratch buffer access + amd::Monitor queue_lock_; //!< Queue lock for access + QueueRecycleInfo() : counter_(1), engineType_(Pal::EngineTypeCompute), index_(0) {} + }; + //! Locks any access to the virtual GPUs class ScopedLockVgpus : public amd::StackObject { public: @@ -239,10 +247,10 @@ class Device : public NullDevice { }; struct ScratchBuffer : public amd::HeapObject { - Memory* memObj_; //!< Memory objects for scratch buffers - uint64_t offset_; //!< Offset from the global scratch store - uint64_t size_; //!< Scratch buffer size on this queue - uint64_t privateMemSize_; //!< Private memory size per thread, allowed by the current scratch + Memory* memObj_; //!< Memory objects for scratch buffers + uint64_t offset_; //!< Offset from the global scratch store + uint64_t size_; //!< Scratch buffer size on this queue + uint64_t privateMemSize_; //!< Private memory size per thread, allowed by the current scratch //! Default constructor ScratchBuffer() : memObj_(nullptr), offset_(0), size_(0), privateMemSize_(0) {} @@ -343,8 +351,7 @@ class Device : public NullDevice { //! Validates kernel before execution virtual bool validateKernel(const amd::Kernel& kernel, //!< AMD kernel object - const device::VirtualDevice* vdev, - bool coop_group = false); + const device::VirtualDevice* vdev, bool coop_group = false); virtual bool SetClockMode(const cl_set_device_clock_mode_input_amd setClockModeInput, cl_set_device_clock_mode_output_amd* pSetClockModeOutput); @@ -530,6 +537,10 @@ class Device : public NullDevice { bool AcquireExclusiveGpuAccess(); void ReleaseExclusiveGpuAccess(VirtualGPU& vgpu) const; + //! Returns PAL Queue pool for recycling + std::map& QueuePool() { return queue_pool_; } + const std::map& QueuePool() const { return queue_pool_; } + private: static void PAL_STDCALL PalDeveloperCallback(void* pPrivateData, const Pal::uint32 deviceIndex, Pal::Developer::CallbackType type, void* pCbData); @@ -554,9 +565,9 @@ class Device : public NullDevice { ) const; //! Allocates/reallocates the scratch buffer, according to the usage - bool allocScratch(uint regNum, //!< Number of the scratch registers - const VirtualGPU* vgpu, //!< Virtual GPU for the allocation - uint vgprs //!< Used VGPRs in the kernel + bool allocScratch(uint regNum, //!< Number of the scratch registers + const VirtualGPU* vgpu, //!< Virtual GPU for the allocation + uint vgprs //!< Used VGPRs in the kernel ); //! Interop for D3D devices @@ -603,7 +614,8 @@ class Device : public NullDevice { std::unordered_set* resourceList_; //!< Active resource list RgpCaptureMgr* rgpCaptureMgr_; //!< RGP capture manager Pal::GpuMemoryHeapProperties - heaps_[Pal::GpuHeapCount]; //!< Information about heaps, returned from PAL + heaps_[Pal::GpuHeapCount]; //!< Information about heaps, returned from PAL + std::map queue_pool_; //!< Pool of PAL queues for recycling }; /*@}*/ // namespace pal diff --git a/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp b/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp index 281d307de3..5e3e4dda5b 100644 --- a/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp @@ -34,6 +34,14 @@ namespace pal { +uint32_t VirtualGPU::Queue::AllocedQueues(const VirtualGPU& gpu, Pal::EngineType type) { + uint32_t allocedQueues = 0; + for (const auto& queue : gpu.dev().QueuePool()) { + allocedQueues += (queue.second->engineType_ == type) ? 1 : 0; + } + return allocedQueues; +} + VirtualGPU::Queue* VirtualGPU::Queue::Create(const VirtualGPU& gpu, Pal::QueueType queueType, uint engineIdx, Pal::ICmdAllocator* cmdAllocator, uint rtCU, amd::CommandQueue::Priority priority, @@ -102,9 +110,49 @@ VirtualGPU::Queue* VirtualGPU::Queue::Create(const VirtualGPU& gpu, Pal::QueueTy VirtualGPU::Queue* queue = new (allocSize) VirtualGPU::Queue(gpu, palDev, residency_limit, max_command_buffers); if (queue != nullptr) { - address addrQ = reinterpret_cast
(&queue[1]); - // Create PAL queue object - result = palDev->CreateQueue(qCreateInfo, addrQ, &queue->iQueue_); + address addrQ = nullptr; + if ((qCreateInfo.engineType == Pal::EngineTypeCompute) || + (qCreateInfo.engineType == Pal::EngineTypeDma)) { + uint32_t index = AllocedQueues(gpu, qCreateInfo.engineType); + // Create PAL queue object + if (index < GPU_MAX_HW_QUEUES) { + Device::QueueRecycleInfo* info = new (qSize) Device::QueueRecycleInfo(); + addrQ = reinterpret_cast
(&info[1]); + result = palDev->CreateQueue(qCreateInfo, addrQ, &queue->iQueue_); + if (result == Pal::Result::Success) { + const_cast(gpu.dev()).QueuePool().insert({queue->iQueue_, info}); + info->engineType_ = qCreateInfo.engineType; + // Save uniqueue index for scratch buffer access + info->index_ = index; + } + } else { + int usage = std::numeric_limits::max(); + uint indexBase = std::numeric_limits::max(); + // Loop through all allocated queues and find the lowest usage + for (const auto& it : gpu.dev().QueuePool()) { + if ((qCreateInfo.engineType == it.second->engineType_) && + (it.second->counter_ <= usage)) { + if ((it.second->counter_ < usage) || + // Preserve the order of allocations, because SDMA engines + // should be used in round-robin manner + ((it.second->counter_ == usage) && (it.second->index_ < indexBase))) { + queue->iQueue_ = it.first; + usage = it.second->counter_; + indexBase = it.second->index_; + } + } + } + // Increment the usage of the current queue + gpu.dev().QueuePool().find(queue->iQueue_)->second->counter_++; + } + Device::QueueRecycleInfo* info = gpu.dev().QueuePool().find(queue->iQueue_)->second; + queue->lock_ = &info->queue_lock_; + addrQ = reinterpret_cast
(&queue[1]); + } else { + // Exclusive compute path + addrQ = reinterpret_cast
(&queue[1]); + result = palDev->CreateQueue(qCreateInfo, addrQ, &queue->iQueue_); + } if (result != Pal::Result::Success) { delete queue; return nullptr; @@ -141,6 +189,13 @@ VirtualGPU::Queue* VirtualGPU::Queue::Create(const VirtualGPU& gpu, Pal::QueueTy } VirtualGPU::Queue::~Queue() { + if (nullptr != iQueue_) { + // Make sure the queues are idle + // It's unclear why PAL could still have a busy queue + amd::ScopedLock l(lock_); + iQueue_->WaitIdle(); + } + // Remove all memory references std::vector memRef; for (auto it : memReferences_) { @@ -161,7 +216,25 @@ VirtualGPU::Queue::~Queue() { } if (nullptr != iQueue_) { - iQueue_->Destroy(); + // Find if this queue was used in recycling + if (lock_ != nullptr) { + // Release the queue if the counter is 0 + if (--gpu_.dev().QueuePool().find(iQueue_)->second->counter_ == 0) { + iQueue_->Destroy(); + const auto& info = gpu_.dev().QueuePool().find(iQueue_); + // Readjust HW queue index for scratch buffer access + for (auto& queue : gpu_.dev().QueuePool()) { + if ((queue.second->engineType_ == info->second->engineType_) && + (queue.second->index_ > info->second->index_)) { + queue.second->index_--; + } + } + delete gpu_.dev().QueuePool().find(iQueue_)->second; + const_cast(gpu_.dev()).QueuePool().erase(iQueue_); + } + } else { + iQueue_->Destroy(); + } } } @@ -269,8 +342,10 @@ bool VirtualGPU::Queue::flush() { // Submit command buffer to OS Pal::Result result; if (gpu_.rgpCaptureEna()) { + amd::ScopedLock l(lock_); result = gpu_.dev().rgpCaptureMgr()->TimedQueueSubmit(iQueue_, cmdBufIdCurrent_, submitInfo); } else { + amd::ScopedLock l(lock_); result = iQueue_->Submit(submitInfo); } if (Pal::Result::Success != result) { @@ -536,8 +611,8 @@ void VirtualGPU::MemoryDependency::clear(bool all) { } VirtualGPU::DmaFlushMgmt::DmaFlushMgmt(const Device& dev) : cbWorkload_(0), dispatchSplitSize_(0) { - aluCnt_ = dev.properties().gfxipProperties.shaderCore.numSimdsPerCu * - dev.info().simdWidth_ * dev.info().maxComputeUnits_; + aluCnt_ = dev.properties().gfxipProperties.shaderCore.numSimdsPerCu * dev.info().simdWidth_ * + dev.info().maxComputeUnits_; maxDispatchWorkload_ = static_cast(dev.info().maxEngineClockFrequency_) * // find time in us dev.settings().maxWorkloadTime_ * aluCnt_; @@ -779,9 +854,9 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs, createInfo.flags.autoMemoryReuse = false; createInfo.allocInfo[Pal::CommandDataAlloc].allocHeap = Pal::GpuHeapGartUswc; createInfo.allocInfo[Pal::CommandDataAlloc].suballocSize = - VirtualGPU::Queue::MaxCommands * (320 + ((profiling) ? 96 : 0)); + VirtualGPU::Queue::MaxCommands * (320 + ((profiling) ? 96 : 0)); createInfo.allocInfo[Pal::CommandDataAlloc].allocSize = - dev().settings().maxCmdBuffers_ * createInfo.allocInfo[Pal::CommandDataAlloc].suballocSize; + dev().settings().maxCmdBuffers_ * createInfo.allocInfo[Pal::CommandDataAlloc].suballocSize; createInfo.allocInfo[Pal::EmbeddedDataAlloc].allocHeap = Pal::GpuHeapGartUswc; createInfo.allocInfo[Pal::EmbeddedDataAlloc].allocSize = 256 * Ki; @@ -808,15 +883,15 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs, uint max_cmd_buffers = dev().settings().maxCmdBuffers_; if (dev().numComputeEngines()) { - // hwRing_ should be set 0 if forced to have single scratch buffer - hwRing_ = (dev().settings().useSingleScratch_) ? 0 : idx; - - queues_[MainEngine] = - Queue::Create(*this, Pal::QueueTypeCompute, idx, cmdAllocator_, rtCUs, - priority, residency_limit, max_cmd_buffers); + queues_[MainEngine] = Queue::Create(*this, Pal::QueueTypeCompute, idx, cmdAllocator_, rtCUs, + priority, residency_limit, max_cmd_buffers); if (nullptr == queues_[MainEngine]) { return false; } + const auto& info = dev().QueuePool().find(queues_[MainEngine]->iQueue_); + hwRing_ = (info != dev().QueuePool().end()) + ? info->second->index_ + : (index() % dev().numExclusiveComputeEngines()) + GPU_MAX_HW_QUEUES; // Check if device has SDMA engines if (dev().numDMAEngines() != 0 && !dev().settings().disableSdma_) { @@ -984,14 +1059,10 @@ VirtualGPU::~VirtualGPU() { { // Destroy queues if (nullptr != queues_[MainEngine]) { - // Make sure the queues are idle - // It's unclear why PAL could still have a busy queue - queues_[MainEngine]->iQueue_->WaitIdle(); delete queues_[MainEngine]; } if (nullptr != queues_[SdmaEngine]) { - queues_[SdmaEngine]->iQueue_->WaitIdle(); delete queues_[SdmaEngine]; } @@ -1107,9 +1178,9 @@ void VirtualGPU::submitReadMemory(amd::ReadMemoryCommand& vcmd) { amd::Image* image = imageBuffer->owner()->asImage(); amd::Coord3D offs(0); // Copy memory from the original image buffer into the backing store image - result = blitMgr().copyBufferToImage(*buffer, *imageBuffer->CopyImageBuffer(), - offs, offs, image->getRegion(), true, - image->getRowPitch(), image->getSlicePitch()); + result = blitMgr().copyBufferToImage(*buffer, *imageBuffer->CopyImageBuffer(), offs, + offs, image->getRegion(), true, + image->getRowPitch(), image->getSlicePitch()); } } } @@ -2177,13 +2248,13 @@ void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) { if (vcmd.cooperativeGroups()) { uint32_t workgroups = 0; for (uint i = 0; i < vcmd.sizes().dimensions(); i++) { - if ((vcmd.sizes().local()[i] != 0) && (vcmd.sizes().global()[i] != 1)) { + if ((vcmd.sizes().local()[i] != 0) && (vcmd.sizes().global()[i] != 1)) { workgroups += (vcmd.sizes().global()[i] / vcmd.sizes().local()[i]); } } uint32_t counter = workgroups * - amd::alignUp(vcmd.sizes().local().product(), dev().info().wavefrontWidth_) / - dev().info().wavefrontWidth_; + amd::alignUp(vcmd.sizes().local().product(), dev().info().wavefrontWidth_) / + dev().info().wavefrontWidth_; bool test = true; VirtualGPU* queue = (test) ? this : dev().xferQueue(); @@ -2199,8 +2270,8 @@ void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) { queue->addBarrier(RgpSqqtBarrierReason::PostDeviceEnqueue); // Submit kernel to HW - if (!queue->submitKernelInternal(vcmd.sizes(), vcmd.kernel(), vcmd.parameters(), - false, &vcmd.event(), vcmd.sharedMemBytes(), + if (!queue->submitKernelInternal(vcmd.sizes(), vcmd.kernel(), vcmd.parameters(), false, + &vcmd.event(), vcmd.sharedMemBytes(), vcmd.cooperativeGroups())) { vcmd.setStatus(CL_INVALID_OPERATION); } @@ -2420,10 +2491,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const amd::Image* image = imageBuffer->owner()->asImage(); amd::Coord3D offs(0); // Copy memory from the the backing store image into original buffer - bool result = blitMgr().copyImageToBuffer( - *imageBuffer->CopyImageBuffer(), *buffer, offs, offs, - image->getRegion(), true, - image->getRowPitch(), image->getSlicePitch()); + bool result = blitMgr().copyImageToBuffer(*imageBuffer->CopyImageBuffer(), *buffer, offs, + offs, image->getRegion(), true, + image->getRowPitch(), image->getSlicePitch()); } wrtBackImageBuffer_.clear(); } @@ -2761,7 +2831,11 @@ void VirtualGPU::submitMakeBuffersResident(amd::MakeBuffersResidentCommand& vcmd dev().iDev()->AddGpuMemoryReferences(numObjects, pGpuMemRef, queues_[MainEngine]->iQueue_, Pal::GpuMemoryRefCantTrim); - dev().iDev()->InitBusAddressableGpuMemory(queues_[MainEngine]->iQueue_, numObjects, pGpuMems); + { + amd::ScopedLock l(queues_[MainEngine]->lock_); + dev().iDev()->InitBusAddressableGpuMemory(queues_[MainEngine]->iQueue_, numObjects, pGpuMems); + } + if (numObjects != 0) { dev().iDev()->RemoveGpuMemoryReferences(numObjects, &pGpuMems[0], queues_[MainEngine]->iQueue_); } @@ -3297,8 +3371,8 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p amd::Coord3D offs(0); // Copy memory from the original image buffer into the backing store image bool result = blitMgr().copyBufferToImage( - *buffer, *imageBuffer->CopyImageBuffer(), offs, offs, - image->getRegion(), true, image->getRowPitch(), image->getSlicePitch()); + *buffer, *imageBuffer->CopyImageBuffer(), offs, offs, image->getRegion(), true, + image->getRowPitch(), image->getSlicePitch()); // Make sure the copy operation is done addBarrier(RgpSqqtBarrierReason::MemDependency); // Use backing store SRD as the replacment diff --git a/projects/clr/rocclr/runtime/device/pal/palvirtual.hpp b/projects/clr/rocclr/runtime/device/pal/palvirtual.hpp index 71e7cf2e73..01103ccd77 100644 --- a/projects/clr/rocclr/runtime/device/pal/palvirtual.hpp +++ b/projects/clr/rocclr/runtime/device/pal/palvirtual.hpp @@ -63,7 +63,8 @@ class VirtualGPU : public device::VirtualDevice { Queue(const VirtualGPU& gpu, Pal::IDevice* iDev, uint64_t residency_limit, uint max_command_buffers) - : iQueue_(nullptr), + : lock_(nullptr), + iQueue_(nullptr), iCmdBuffs_(max_command_buffers, nullptr), iCmdFences_(max_command_buffers, nullptr), last_kernel_(nullptr), @@ -149,6 +150,9 @@ class VirtualGPU : public device::VirtualDevice { uint cmdBufId() const { return cmdBufIdCurrent_; } + static uint32_t AllocedQueues(const VirtualGPU& gpu, Pal::EngineType type); + + amd::Monitor* lock_; //!< Lock PAL queue for access Pal::IQueue* iQueue_; //!< PAL queue object std::vector iCmdBuffs_; //!< PAL command buffers std::vector iCmdFences_; //!< PAL fences, associated with CMD @@ -205,7 +209,7 @@ class VirtualGPU : public device::VirtualDevice { uint profileEnabled_ : 1; //!< Profiling is enabled for WaveLimiter uint perfCounterEnabled_ : 1; //!< PerfCounter is enabled uint rgpCaptureEnabled_ : 1; //!< RGP capture is enabled in the runtime - uint imageBufferWrtBack_: 1; //!< Enable image buffer write back + uint imageBufferWrtBack_ : 1; //!< Enable image buffer write back }; uint value_; State() : value_(0) {} @@ -640,11 +644,11 @@ class VirtualGPU : public device::VirtualDevice { uint deviceQueueSize_; //!< Device queue size uint maskGroups_; //!< The number of mask groups processed in the scheduler by one thread - Memory* hsaQueueMem_; //!< Memory for the amd_queue_t object - Pal::ICmdAllocator* cmdAllocator_; //!< Command buffer allocator - Queue* queues_[AllEngines]; //!< HW queues for all engines - MemoryRange sdmaRange_; //!< SDMA memory range for write access - std::vector wrtBackImageBuffer_; //!< Array of images for write back + Memory* hsaQueueMem_; //!< Memory for the amd_queue_t object + Pal::ICmdAllocator* cmdAllocator_; //!< Command buffer allocator + Queue* queues_[AllEngines]; //!< HW queues for all engines + MemoryRange sdmaRange_; //!< SDMA memory range for write access + std::vector wrtBackImageBuffer_; //!< Array of images for write back }; inline void VirtualGPU::addVmMemory(const Memory* memory) {