From 6e1e293bd76cae2faf3f1b6b51e3ef550b710b3e Mon Sep 17 00:00:00 2001 From: foreman Date: Fri, 23 Sep 2016 13:29:09 -0400 Subject: [PATCH] P4 to Git Change 1318204 by gandryey@gera-w8 on 2016/09/23 13:14:59 SWDEV-86035 - Add PAL backend to OpenCL - Avoid malloc()/free() calls during the dispatch Affected files ... ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpudevice.hpp#161 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.hpp#9 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#23 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.hpp#10 edit --- rocclr/runtime/device/gpu/gpudevice.hpp | 2 +- rocclr/runtime/device/pal/paldevice.hpp | 2 +- rocclr/runtime/device/pal/palvirtual.cpp | 102 +++++++++++++---------- rocclr/runtime/device/pal/palvirtual.hpp | 28 +++++-- 4 files changed, 82 insertions(+), 52 deletions(-) diff --git a/rocclr/runtime/device/gpu/gpudevice.hpp b/rocclr/runtime/device/gpu/gpudevice.hpp index 985eb079f6..3195289f75 100644 --- a/rocclr/runtime/device/gpu/gpudevice.hpp +++ b/rocclr/runtime/device/gpu/gpudevice.hpp @@ -478,7 +478,7 @@ public: typedef std::vector VirtualGPUs; //! Returns the list of all virtual GPUs running on this device - const VirtualGPUs vgpus() const { return vgpus_; } + const VirtualGPUs& vgpus() const { return vgpus_; } VirtualGPUs vgpus_; //!< The list of all running virtual gpus (lock protected) //! Scratch buffer allocation diff --git a/rocclr/runtime/device/pal/paldevice.hpp b/rocclr/runtime/device/pal/paldevice.hpp index 6bbd1bc50d..f689ea0646 100644 --- a/rocclr/runtime/device/pal/paldevice.hpp +++ b/rocclr/runtime/device/pal/paldevice.hpp @@ -386,7 +386,7 @@ public: typedef std::vector VirtualGPUs; //! Returns the list of all virtual GPUs running on this device - const VirtualGPUs vgpus() const { return vgpus_; } + const VirtualGPUs& vgpus() const { return vgpus_; } VirtualGPUs vgpus_; //!< The list of all running virtual gpus (lock protected) //! Scratch buffer allocation diff --git a/rocclr/runtime/device/pal/palvirtual.cpp b/rocclr/runtime/device/pal/palvirtual.cpp index b88737cb81..b81f4db604 100644 --- a/rocclr/runtime/device/pal/palvirtual.cpp +++ b/rocclr/runtime/device/pal/palvirtual.cpp @@ -170,7 +170,7 @@ VirtualGPU::Queue::submit(bool forceFlush) bool VirtualGPU::Queue::flush() { - std::vector memRefs; + palMemRefs_.resize(0); // Stop commands building if (Pal::Result::Success != iCmdBuffs_[cmdBufIdSlot_]->End()) { LogError("PAL failed to finalize a command buffer!"); @@ -182,12 +182,12 @@ VirtualGPU::Queue::flush() it->second &= ~FirstMemoryReference; Pal::GpuMemoryRef memRef = {}; memRef.pGpuMemory = it->first; - memRefs.push_back(memRef); + palMemRefs_.push_back(memRef); } } - if (memRefs.size() != 0) { - iDev_->AddGpuMemoryReferences(memRefs.size(), &memRefs[0], iQueue_, + if (palMemRefs_.size() != 0) { + iDev_->AddGpuMemoryReferences(palMemRefs_.size(), &palMemRefs_[0], iQueue_, Pal::GpuMemoryRefCantTrim); } @@ -257,20 +257,19 @@ VirtualGPU::Queue::flush() return false; } - memRefs.clear(); - std::vector iMems; + palMems_.resize(0); // Remove old memory references for (auto it = memReferences_.begin(); it != memReferences_.end();) { if (it->second == cmdBufIdSlot_) { - iMems.push_back(it->first); + palMems_.push_back(it->first); it = memReferences_.erase(it); } else { ++it; } } - if (iMems.size() != 0) { - iDev_->RemoveGpuMemoryReferences(iMems.size(), &iMems[0], iQueue_); + if (palMems_.size() != 0) { + iDev_->RemoveGpuMemoryReferences(palMems_.size(), &palMems_[0], iQueue_); } return true; @@ -484,7 +483,7 @@ VirtualGPU::addXferWrite(Memory& memory) { if (xferWriteBuffers_.size() > 7) { dev().xferWrite().release(*this, *xferWriteBuffers_.front()); - xferWriteBuffers_.pop_front(); + xferWriteBuffers_.erase(xferWriteBuffers_.begin()); } // Delay destruction @@ -497,7 +496,7 @@ VirtualGPU::releaseXferWrite() for (auto& memory : xferWriteBuffers_) { dev().xferWrite().release(*this, *memory); } - xferWriteBuffers_.clear(); + xferWriteBuffers_.resize(0); } void @@ -506,7 +505,7 @@ VirtualGPU::addPinnedMem(amd::Memory* mem) if (nullptr == findPinnedMem(mem->getHostMem(), mem->getSize())) { if (pinnedMems_.size() > 7) { pinnedMems_.front()->release(); - pinnedMems_.pop_front(); + pinnedMems_.erase(pinnedMems_.begin()); } // Start operation, since we should release mem object @@ -523,7 +522,7 @@ VirtualGPU::releasePinnedMem() for (auto& amdMemory : pinnedMems_) { amdMemory->release(); } - pinnedMems_.clear(); + pinnedMems_.resize(0); } amd::Memory* @@ -876,6 +875,12 @@ VirtualGPU::~VirtualGPU() static const bool SkipScratch = false; releaseMemObjects(SkipScratch); + while (!freeCbList_.empty()) { + auto cb = freeCbList_.top(); + delete cb; + freeCbList_.pop(); + } + // Destroy printf object delete printfDbg_; @@ -1892,7 +1897,7 @@ VirtualGPU::submitKernelInternal( // Get the HSA kernel object const HSAILKernel& hsaKernel = static_cast(*(kernel.getDeviceKernel(dev()))); - std::vector memList; + dispMemList_.resize(0); bool printfEnabled = (hsaKernel.printfInfo().size() > 0) ? true:false; if (!printfDbgHSA().init(*this, printfEnabled )) { @@ -1901,7 +1906,7 @@ VirtualGPU::submitKernelInternal( } // Check memory dependency and SVM objects - if (!processMemObjectsHSA(kernel, parameters, nativeMem, &memList)) { + if (!processMemObjectsHSA(kernel, parameters, nativeMem)) { LogError("Wrong memory objects!"); return false; } @@ -1927,9 +1932,9 @@ VirtualGPU::submitKernelInternal( vmDefQueue = gpuDefQueue->virtualQueue_->vmAddress(); // Add memory handles before the actual dispatch - memList.push_back(gpuDefQueue->virtualQueue_); - memList.push_back(gpuDefQueue->schedParams_); - memList.push_back(hsaKernel.prog().kernelTable()); + dispMemList_.push_back(gpuDefQueue->virtualQueue_); + dispMemList_.push_back(gpuDefQueue->schedParams_); + dispMemList_.push_back(hsaKernel.prog().kernelTable()); gpuDefQueue->writeVQueueHeader(*this, hsaKernel.prog().kernelTable()->vmAddress()); } @@ -1985,7 +1990,7 @@ VirtualGPU::submitKernelInternal( // Program the kernel arguments for the GPU execution hsa_kernel_dispatch_packet_t* aqlPkt = hsaKernel.loadArguments(*this, kernel, tmpSizes, parameters, nativeMem, - vmDefQueue, &vmParentWrap, memList); + vmDefQueue, &vmParentWrap, dispMemList_); if (nullptr == aqlPkt) { LogError("Couldn't load kernel arguments"); return false; @@ -1995,12 +2000,12 @@ VirtualGPU::submitKernelInternal( // Check if the device allocated more registers than the old setup if (hsaKernel.workGroupInfo()->scratchRegs_ > 0) { scratch = dev().scratch(hwRing()); - memList.push_back(scratch->memObj_); + dispMemList_.push_back(scratch->memObj_); } // Add GSL handle to the memory list for VidMM - for (uint i = 0; i < memList.size(); ++i) { - addVmMemory(memList[i]); + for (uint i = 0; i < dispMemList_.size(); ++i) { + addVmMemory(dispMemList_[i]); } // HW Debug for the kernel? @@ -2165,7 +2170,7 @@ VirtualGPU::submitKernelInternal( param->scratch = scratchBuf->vmAddress(); param->numMaxWaves = 32 * dev().info().maxComputeUnits_; param->scratchOffset = dev().scratch(gpuDefQueue->hwRing())->offset_; - memList.push_back(scratchBuf); + dispMemList_.push_back(scratchBuf); } else { param->numMaxWaves = 0; @@ -2176,11 +2181,11 @@ VirtualGPU::submitKernelInternal( // Add all kernels in the program to the mem list. //! \note Runtime doesn't know which one will be called - hsaKernel.prog().fillResListWithKernels(memList); + hsaKernel.prog().fillResListWithKernels(dispMemList_); // Add GPU memory handle to the memory list for VidMM - for (uint i = 0; i < memList.size(); ++i) { - gpuDefQueue->addVmMemory(memList[i]); + for (uint i = 0; i < dispMemList_.size(); ++i) { + gpuDefQueue->addVmMemory(dispMemList_[i]); } Pal::gpusize signalAddr = gpuDefQueue->schedParams_->vmAddress() + @@ -2195,8 +2200,8 @@ VirtualGPU::submitKernelInternal( gpuDefQueue->eventEnd(MainEngine, gpuEvent, ForceSubmitFirst); // Set GPU event for the used resources - for (uint i = 0; i < memList.size(); ++i) { - memList[i]->setBusy(*gpuDefQueue, gpuEvent); + for (uint i = 0; i < dispMemList_.size(); ++i) { + dispMemList_[i]->setBusy(*gpuDefQueue, gpuEvent); } if (dev().settings().useDeviceQueue_) { @@ -2218,8 +2223,8 @@ VirtualGPU::submitKernelInternal( } // Set GPU event for the used resources - for (uint i = 0; i < memList.size(); ++i) { - memList[i]->setBusy(*this, gpuEvent); + for (uint i = 0; i < dispMemList_.size(); ++i) { + dispMemList_[i]->setBusy(*this, gpuEvent); } // Update the global GPU event @@ -2253,13 +2258,13 @@ VirtualGPU::submitMarker(amd::Marker& vcmd) // Loop through all outstanding command batches while (!cbList_.empty()) { - CommandBatchList::const_iterator it = cbList_.begin(); + auto cb = cbList_.top(); // Wait for completion - foundEvent = awaitCompletion(*it, vcmd.waitingEvent()); + foundEvent = awaitCompletion(cb, vcmd.waitingEvent()); // Release a command batch - delete *it; + freeCbList_.push(cb); // Remove command batch from the list - cbList_.pop_front(); + cbList_.pop(); // Early exit if we found a command if (foundEvent) break; } @@ -2724,7 +2729,17 @@ VirtualGPU::flush(amd::Command* list, bool wait) // Insert the current batch into a list if (nullptr != list) { - cb = new CommandBatch(list, cal()->events_, cal()->lastTS_); + if (!freeCbList_.empty()) { + cb = freeCbList_.top(); + } + + if (nullptr == cb) { + cb = new CommandBatch(list, cal()->events_, cal()->lastTS_); + } + else { + freeCbList_.pop(); + cb->init(list, cal()->events_, cal()->lastTS_); + } } { @@ -2743,25 +2758,25 @@ VirtualGPU::flush(amd::Command* list, bool wait) // Mark last TS as nullptr, so runtime won't process empty batches with the old TS cal_.lastTS_ = nullptr; if (nullptr != cb) { - cbList_.push_back(cb); + cbList_.push(cb); } wait |= state_.forceWait_; // Loop through all outstanding command batches while (!cbList_.empty()) { - CommandBatchList::const_iterator it = cbList_.begin(); + auto cb = cbList_.top(); // Check if command batch finished without a wait bool finished = true; for (uint i = 0; i < AllEngines; ++i) { - finished &= isDone(&(*it)->events_[i]); + finished &= isDone(&cb->events_[i]); } if (finished || wait) { // Wait for completion - awaitCompletion(*it); + awaitCompletion(cb); // Release a command batch - delete *it; + freeCbList_.push(cb); // Remove command batch from the list - cbList_.pop_front(); + cbList_.pop(); } else { // Early exit if no finished @@ -3066,8 +3081,7 @@ bool VirtualGPU::processMemObjectsHSA( const amd::Kernel& kernel, const_address params, - bool nativeMem, - std::vector* memList) + bool nativeMem) { static const bool NoAlias = true; const HSAILKernel& hsaKernel = static_cast @@ -3129,7 +3143,7 @@ VirtualGPU::processMemObjectsHSA( // Validate SVM passed in the non argument list memoryDependency().validate(*this, gpuMemory, IsReadOnly); - memList->push_back(gpuMemory); + dispMemList_.push_back(gpuMemory); } else { return false; diff --git a/rocclr/runtime/device/pal/palvirtual.hpp b/rocclr/runtime/device/pal/palvirtual.hpp index d4716f0550..fe1cc33c80 100644 --- a/rocclr/runtime/device/pal/palvirtual.hpp +++ b/rocclr/runtime/device/pal/palvirtual.hpp @@ -3,6 +3,7 @@ // #pragma once +#include #include "device/pal/paldefs.hpp" #include "device/pal/palconstbuf.hpp" #include "device/pal/palprintf.hpp" @@ -105,6 +106,8 @@ public: uint cmdCnt_; //!< Counter of commands std::map memReferences_; Util::VirtualLinearAllocator vlAlloc_; + std::vector palMemRefs_; + std::vector palMems_; }; struct CommandBatch : public amd::HeapObject @@ -118,10 +121,22 @@ public: amd::Command* head, //!< Command batch head const GpuEvent* events, //!< HW events on all engines TimeStamp* lastTS //!< Last TS in command batch - ): head_(head), lastTS_(lastTS) + ) { + init(head, events, lastTS); + } + + void init( + amd::Command* head, //!< Command batch head + const GpuEvent* events, //!< HW events on all engines + TimeStamp* lastTS //!< Last TS in command batch + ) + { + head_ = head; + lastTS_ = lastTS; memcpy(&events_, events, AllEngines * sizeof(GpuEvent)); } + }; //! The virtual GPU states @@ -511,8 +526,7 @@ private: bool processMemObjectsHSA( const amd::Kernel& kernel, //!< AMD kernel object for execution const_address params, //!< Pointer to the param's store - bool nativeMem, //!< Native memory objects - std::vector* memList //!< Memory list for KMD tracking + bool nativeMem //!< Native memory objects ); //! Common function for fill memory used by both svm Fill and non-svm fill @@ -563,11 +577,12 @@ private: DmaFlushMgmt dmaFlushMgmt_; //!< DMA flush management - std::list xferWriteBuffers_; //!< Stage write buffers - std::list pinnedMems_;//!< Pinned memory list + std::vector xferWriteBuffers_; //!< Stage write buffers + std::vector pinnedMems_;//!< Pinned memory list - typedef std::list CommandBatchList; + typedef std::stack CommandBatchList; CommandBatchList cbList_; //!< List of command batches + CommandBatchList freeCbList_; //!< List of command batches uint hwRing_; //!< HW ring used on this virtual device @@ -585,6 +600,7 @@ private: Pal::ICmdAllocator* cmdAllocator_; //!< Command buffer allocator Queue* queues_[AllEngines]; //!< HW queues for all engines MemoryRange sdmaRange_; //!< SDMA memory range for write access + std::vector dispMemList_; //!< Memory list of all mem objects used in the disaptch }; /*@}*/} // namespace pal