From 738dcbcdd4422540ae522804201c4318407b6ae8 Mon Sep 17 00:00:00 2001 From: foreman Date: Tue, 4 Apr 2017 11:09:12 -0500 Subject: [PATCH] P4 to Git Change 1394146 by gandryey@gera-w8 on 2017/04/04 11:59:55 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SWDEV-79445 - OCL generic changes and code clean-up - Added a simple hang analysis (PAL path only). GPU_ANALYZE_HANG controls the feature. It will create ocl_hang_dump.txt, which will contain OCL resident resources used in the last submissions on this queue and last kernel name. ItÂ’s recommended to use this feature with GPU_FLUSH_ON_EXECUTION or AMD_OCL_WAIT_COMMAND Affected files ... ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#46 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.hpp#20 edit ... //depot/stg/opencl/drivers/opencl/runtime/utils/flags.hpp#270 edit --- rocclr/runtime/device/pal/palvirtual.cpp | 64 ++++++++++++++++++++++-- rocclr/runtime/device/pal/palvirtual.hpp | 41 +++++++++------ rocclr/runtime/utils/flags.hpp | 2 + 3 files changed, 89 insertions(+), 18 deletions(-) diff --git a/rocclr/runtime/device/pal/palvirtual.cpp b/rocclr/runtime/device/pal/palvirtual.cpp index fc3594abe5..efbe955678 100644 --- a/rocclr/runtime/device/pal/palvirtual.cpp +++ b/rocclr/runtime/device/pal/palvirtual.cpp @@ -360,6 +360,57 @@ VirtualGPU::Queue::isDone(uint id) return true; } +void +VirtualGPU::Queue::DumpMemoryReferences() const +{ + std::fstream dump; + std::stringstream file_name("ocl_hang_dump.txt"); + uint64_t start = amd::Os::timeNanos() / 1e9; + + dump.open(file_name.str().c_str(), (std::fstream::out | std::fstream::app)); + // Check if we have OpenCL program + if (dump.is_open()) { + dump << start << " Queue: "; + switch (iQueue_->Type()) { + case Pal::QueueTypeCompute: + dump << "Compute"; + break; + case Pal::QueueTypeDma: + dump << "SDMA"; + break; + default: + dump << "unknown"; + break; + } + dump << "\n" << "Resident memory resources:\n"; + uint idx = 0; + for (auto it : memReferences_) { + dump << " " << idx << "\t["; + dump.setf(std::ios::hex, std::ios::basefield); + dump.setf(std::ios::showbase); + dump << (it.first)->Desc().gpuVirtAddr << ", " << + (it.first)->Desc().gpuVirtAddr + (it.first)->Desc().size; + dump.setf(std::ios::dec); + dump << "] CbId:" << it.second << "\n"; + idx++; + } + } + if (last_kernel_ != nullptr) { + const amd::KernelSignature& signature = last_kernel_->signature(); + const amd::KernelParameters& params = last_kernel_->parameters(); + dump << last_kernel_->name() << std::endl; + for (size_t i = 0; i < signature.numParameters(); ++i) { + const amd::KernelParameterDescriptor& desc = signature.at(i); + // Find if the current argument is a memory object + if ((desc.type_ == T_POINTER) && + (desc.addressQualifier_ != CL_KERNEL_ARG_ADDRESS_LOCAL)) { + dump << " " << desc.name_ << ": " << std::endl; + } + } + } + dump.close(); +} + bool VirtualGPU::MemoryDependency::create(size_t numMemObj) { @@ -1998,6 +2049,8 @@ VirtualGPU::submitKernelInternal( VirtualGPU* gpuDefQueue = nullptr; amd::HwDebugManager * dbgManager = dev().hwDebugMgr(); + AddKernel(kernel); + // Get the HSA kernel object const HSAILKernel& hsaKernel = static_cast(*(kernel.getDeviceKernel(dev()))); @@ -3177,18 +3230,21 @@ VirtualGPU::profilingCollectResults(CommandBatch* cb, const amd::Event* waitingE return found; } -bool +void VirtualGPU::addVmMemory(const Memory* memory) { queues_[MainEngine]->addCmdMemRef(memory->iMem()); - return true; +} +void +VirtualGPU::AddKernel(const amd::Kernel& kernel) const +{ + queues_[MainEngine]->last_kernel_ = &kernel; } -bool +void VirtualGPU::addDoppRef(const Memory* memory, bool lastDoppCmd, bool pfpaDoppCmd) { queues_[MainEngine]->addCmdDoppRef(memory->iMem(), lastDoppCmd, pfpaDoppCmd); - return true; } void diff --git a/rocclr/runtime/device/pal/palvirtual.hpp b/rocclr/runtime/device/pal/palvirtual.hpp index 1fc6494b30..66295988f1 100644 --- a/rocclr/runtime/device/pal/palvirtual.hpp +++ b/rocclr/runtime/device/pal/palvirtual.hpp @@ -50,6 +50,9 @@ public: static const uint64_t WaitTimeoutInNsec = 6000000000; static const uint64_t PollIntervalInNsec = 200000; + Queue(const Queue&) = delete; + Queue& operator=(const Queue&) = delete; + static Queue* Create( Pal::IDevice* palDev, //!< PAL device object Pal::QueueType queueType, //!< PAL queue type @@ -60,13 +63,13 @@ public: ); Queue(Pal::IDevice* palDev) - : iQueue_(NULL), iDev_(palDev), + : iQueue_(nullptr), last_kernel_(nullptr), iDev_(palDev), cmdBufIdSlot_(StartCmdBufIdx), cmdBufIdCurrent_(StartCmdBufIdx), cmbBufIdRetired_(0), cmdCnt_(0), vlAlloc_(64 * Ki) { for (uint i = 0; i < MaxCmdBuffers; ++i) { - iCmdBuffs_[i] = NULL; - iCmdFences_[i] = NULL; + iCmdBuffs_[i] = nullptr; + iCmdFences_[i] = nullptr; } vlAlloc_.Init(); } @@ -82,12 +85,12 @@ public: { Pal::GpuMemoryRef memRef = {}; memRef.pGpuMemory = iMem; - iDev_->AddGpuMemoryReferences(1, &memRef, NULL, + iDev_->AddGpuMemoryReferences(1, &memRef, nullptr, Pal::GpuMemoryRefCantTrim); } void removeMemRef(Pal::IGpuMemory* iMem) const { - iDev_->RemoveGpuMemoryReferences(1, &iMem, NULL); + iDev_->RemoveGpuMemoryReferences(1, &iMem, nullptr); } // ibReuse forces event wait without polling, to make sure event occured @@ -112,6 +115,9 @@ public: else if ((Pal::Result::NotReady == result) || (Pal::Result::Timeout == result)) { LogWarning("PAL fence isn't ready!"); + if (GPU_ANALYZE_HANG) { + DumpMemoryReferences(); + } } else { LogError("PAL wait for a fence failed!"); @@ -134,10 +140,12 @@ public: Pal::ICmdBuffer* iCmd() const { return iCmdBuffs_[cmdBufIdSlot_]; } Pal::IQueue* iQueue_; //!< PAL queue object - Pal::ICmdBuffer* iCmdBuffs_[MaxCmdBuffers]; //!< PAL command buffers - Pal::IFence* iCmdFences_[MaxCmdBuffers]; //!< PAL fences, associated with CMD + Pal::ICmdBuffer* iCmdBuffs_[MaxCmdBuffers]; //!< PAL command buffers + Pal::IFence* iCmdFences_[MaxCmdBuffers];//!< PAL fences, associated with CMD + const amd::Kernel* last_kernel_; //!< Last submitted kernel private: + void DumpMemoryReferences() const; Pal::IDevice* iDev_; //!< PAL device uint cmdBufIdSlot_; //!< Command buffer ID slot for submissions uint cmdBufIdCurrent_; //!< Current global command buffer ID @@ -207,7 +215,7 @@ public: public: //! Default constructor MemoryDependency() - : memObjectsInQueue_(NULL) + : memObjectsInQueue_(nullptr) , numMemObjectsInQueue_(0) , maxMemObjectsInQueue_(0) {} @@ -294,7 +302,7 @@ public: const amd::Kernel& kernel, //!< Kernel for execution const_address parameters, //!< Parameters for the kernel bool nativeMem = true, //!< Native memory objects - amd::Event* enqueueEvent = NULL //!< Event provided in the enqueue kernel command + amd::Event* enqueueEvent = nullptr //!< Event provided in the enqueue kernel command ); void submitNativeFn(amd::NativeFnCommand& vcmd); void submitFillMemory(amd::FillMemoryCommand& vcmd); @@ -316,7 +324,7 @@ public: void releaseMemory(Pal::IGpuMemory* iMem, bool wait = true); - void flush(amd::Command* list = NULL, bool wait = false); + void flush(amd::Command* list = nullptr, bool wait = false); bool terminate() { return true; } //! Returns GPU device object associated with this kernel @@ -350,7 +358,7 @@ public: //! Wait for all engines on this Virtual GPU //! Returns TRUE if CPU didn't wait for GPU bool waitAllEngines( - CommandBatch* cb = NULL //!< Command batch + CommandBatch* cb = nullptr //!< Command batch ); //! Waits for the latest GPU event with a lock to prevent multiple entries @@ -382,12 +390,17 @@ public: ); //! Adds a memory handle into the GSL memory array for Virtual Heap - bool addVmMemory( + void addVmMemory( const Memory* memory //!< GPU memory object ); + //! Adds the last submitted kernel to the queue for tracking a possible hang + void AddKernel( + const amd::Kernel& kernel //!< AMD kernel object + ) const; + //! Adds a dopp desktop texture reference - bool addDoppRef( + void addDoppRef( const Memory* memory, //!< GPU memory object bool lastDoopCmd, //!< is the last submission for the pre-present primary bool pfpaDoppCmd //!< is a submission for the pre-present primary @@ -565,7 +578,7 @@ private: //! Awaits a command batch with a waiting event bool awaitCompletion( CommandBatch* cb, //!< Command batch for to wait - const amd::Event* waitingEvent = NULL //!< A waiting event + const amd::Event* waitingEvent = nullptr //!< A waiting event ); //! Detects memory dependency for HSAIL kernels and flushes caches diff --git a/rocclr/runtime/utils/flags.hpp b/rocclr/runtime/utils/flags.hpp index 37e35d539e..145cbcba48 100644 --- a/rocclr/runtime/utils/flags.hpp +++ b/rocclr/runtime/utils/flags.hpp @@ -183,6 +183,8 @@ release(bool, GPU_WAVE_LIMIT_ENABLE, false, \ "1 = Enable adaptive wave limiter") \ release(bool, OCL_STUB_PROGRAMS, false, \ "1 = Enables OCL programs stubing") \ +release(bool, GPU_ANALYZE_HANG, false, \ + "1 = Enables GPU hang analysis") \ release_on_stg(uint, GPU_WAVE_LIMIT_CU_PER_SH, 0, \ "Assume the number of CU per SH for wave limiter") \ release_on_stg(uint, GPU_WAVE_LIMIT_MAX_WAVE, 10, \