From 1a3e93121dbbb238479aaabe26f6b585528f02ed Mon Sep 17 00:00:00 2001 From: foreman Date: Mon, 17 Oct 2016 14:59:22 -0400 Subject: [PATCH] P4 to Git Change 1327404 by gandryey@gera-w8 on 2016/10/17 14:20:58 SWDEV-102761 - [CQE OCL][OpenCL on PAL] Few WF Conformance tests are stuck in middle/hardhang while running using csv file - Disable non-user mode queue. There is a hang and deadlock in OS if a mix of user and non-user queues is used. - Wait for the fence longer if PAL returned a timeout error - Slightly change CB warmup code to avoid PAL overhead on the first submit Affected files ... ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#24 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.cpp#14 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#30 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.hpp#14 edit --- rocclr/runtime/device/pal/paldevice.cpp | 28 +++++++------- rocclr/runtime/device/pal/palresource.cpp | 9 +++-- rocclr/runtime/device/pal/palvirtual.cpp | 47 +++++++++-------------- rocclr/runtime/device/pal/palvirtual.hpp | 28 +++++++++++++- 4 files changed, 66 insertions(+), 46 deletions(-) diff --git a/rocclr/runtime/device/pal/paldevice.cpp b/rocclr/runtime/device/pal/paldevice.cpp index b04f24d3a3..930bcbdab2 100644 --- a/rocclr/runtime/device/pal/paldevice.cpp +++ b/rocclr/runtime/device/pal/paldevice.cpp @@ -782,20 +782,6 @@ Device::create(Pal::IDevice* device) mapCache_->push_back(nullptr); size_t resourceCacheSize = settings().resourceCacheSize_; - -#ifdef DEBUG - std::stringstream message; - if (settings().remoteAlloc_) { - message << "Using *Remote* memory"; - } - else { - message << "Using *Local* memory"; - } - - message << std::endl; - LogInfo(message.str().c_str()); -#endif // DEBUG - // Create resource cache. // \note Cache must be created before any resource creation to avoid nullptr check resourceCache_ = new ResourceCache(resourceCacheSize); @@ -806,6 +792,20 @@ Device::create(Pal::IDevice* device) // Fill the device info structure fillDeviceInfo(properties(), heaps, 16*Ki, numComputeEngines()); +#ifdef DEBUG + std::stringstream message; + message << info_.name_; + if (settings().remoteAlloc_) { + message << ": Using *Remote* memory"; + } + else { + message << ": Using *Local* memory"; + } + + message << std::endl; + LogInfo(message.str().c_str()); +#endif // DEBUG + for (uint i = 0; i < Pal::GpuHeap::GpuHeapCount; ++i) { freeMem[i] = heaps[i].heapSize; } diff --git a/rocclr/runtime/device/pal/palresource.cpp b/rocclr/runtime/device/pal/palresource.cpp index 0b890d9ab6..a49976c29d 100644 --- a/rocclr/runtime/device/pal/palresource.cpp +++ b/rocclr/runtime/device/pal/palresource.cpp @@ -1964,11 +1964,14 @@ Resource::rename(VirtualGPU& gpu, bool force) void Resource::warmUpRenames(VirtualGPU& gpu) { - for (uint i = 0; i < dev().settings().maxRenames_; ++i) { + // Make sure OCL touches every command buffer in the queue to avoid delays on the first submit + uint flush = dev().settings().maxRenames_ / VirtualGPU::Queue::MaxCmdBuffers; + flush = (flush == 0) ? 1 : flush; + for (uint i = 1; i <= dev().settings().maxRenames_; ++i) { uint dummy = 0; - const bool NoWait = false; + const bool Wait = (i % flush == 0) ? true : false; // Write 0 for the buffer paging by VidMM - writeRawData(gpu, 0, sizeof(dummy), &dummy, NoWait); + writeRawData(gpu, 0, sizeof(dummy), &dummy, Wait); const bool Force = true; rename(gpu, Force); } diff --git a/rocclr/runtime/device/pal/palvirtual.cpp b/rocclr/runtime/device/pal/palvirtual.cpp index a6d8e12466..34549e699d 100644 --- a/rocclr/runtime/device/pal/palvirtual.cpp +++ b/rocclr/runtime/device/pal/palvirtual.cpp @@ -187,8 +187,11 @@ VirtualGPU::Queue::flush() } if (palMemRefs_.size() != 0) { - iDev_->AddGpuMemoryReferences(palMemRefs_.size(), &palMemRefs_[0], iQueue_, - Pal::GpuMemoryRefCantTrim); + if (Pal::Result::Success != iDev_->AddGpuMemoryReferences( + palMemRefs_.size(), &palMemRefs_[0], iQueue_, Pal::GpuMemoryRefCantTrim)) { + LogError("PAL failed to make resident resources!"); + return false; + } } Pal::SubmitInfo submitInfo = {}; @@ -202,11 +205,7 @@ VirtualGPU::Queue::flush() return false; } if (GPU_FLUSH_ON_EXECUTION) { - if (Pal::Result::Success != - iDev_->WaitForFences(1, &iCmdFences_[cmdBufIdSlot_], true, WaitTimeoutInNsec)) { - LogError("PAL wait for a fence failed!"); - return false; - } + waifForFence(cmdBufIdSlot_); } // Reset the counter of commands @@ -216,7 +215,8 @@ VirtualGPU::Queue::flush() cmdBufIdCurrent_++; if (cmdBufIdCurrent_ == GpuEvent::InvalidID) { - ///@todo handle wrapping + // Wait for the last one + waifForFence(cmdBufIdSlot_); cmdBufIdCurrent_ = 1; cmbBufIdRetired_ = 0; } @@ -225,13 +225,8 @@ VirtualGPU::Queue::flush() cmdBufIdSlot_ = cmdBufIdCurrent_ % MaxCmdBuffers; // Make sure the slot isn't busy - if (Pal::Result::NotReady == iCmdFences_[cmdBufIdSlot_]->GetStatus()) { - if (Pal::Result::Success != - iDev_->WaitForFences(1, &iCmdFences_[cmdBufIdSlot_], true, WaitTimeoutInNsec)) { - LogError("PAL wait for a fence failed!"); - return false; - } - } + waifForFence(cmdBufIdSlot_); + // Progress retired TS if ((cmdBufIdCurrent_ > MaxCmdBuffers) && (cmbBufIdRetired_ < (cmdBufIdCurrent_ - MaxCmdBuffers))) { @@ -282,18 +277,10 @@ VirtualGPU::Queue::waitForEvent(uint id) return true; } - uint slotId = id % MaxCmdBuffers; - - // Wait for the specified fence - if (Pal::Result::Success != iCmdFences_[slotId]->GetStatus()) { - if (Pal::Result::Success != - iDev_->WaitForFences(1, &iCmdFences_[slotId], true, WaitTimeoutInNsec)) { - LogError("PAL wait for a fence failed!"); - return false; - } - } + uint slotId = id % MaxCmdBuffers; + bool result = waifForFence(slotId); cmbBufIdRetired_ = id; - return true; + return result; } bool @@ -729,13 +716,17 @@ VirtualGPU::create(bool profiling, uint deviceQueueSize) } if (dev().numComputeEngines()) { - uint idx = index() % dev().numComputeEngines(); + //! @todo There is a hang with a mix of user and non user queues. + //! Currently there is no simple way to detect which queue is what. + //! Disable first for now. + const uint firstQueue = (dev().numComputeEngines() > 2) ? 1 : 0; + uint idx = index() % (dev().numComputeEngines() - firstQueue); // hwRing_ should be set 0 if forced to have single scratch buffer hwRing_ = (dev().settings().useSingleScratch_) ? 0 : idx; queues_[MainEngine] = Queue::Create( - dev().iDev(), Pal::QueueTypeCompute, idx, cmdAllocator_); + dev().iDev(), Pal::QueueTypeCompute, idx + firstQueue, cmdAllocator_); if (nullptr == queues_[MainEngine]) { return false; } diff --git a/rocclr/runtime/device/pal/palvirtual.hpp b/rocclr/runtime/device/pal/palvirtual.hpp index 25285fc09f..e68248c68c 100644 --- a/rocclr/runtime/device/pal/palvirtual.hpp +++ b/rocclr/runtime/device/pal/palvirtual.hpp @@ -11,9 +11,11 @@ #include "device/pal/palsched.hpp" #include "device/pal/paldebugger.hpp" #include "device/blit.hpp" +#include "palUtil.h" #include "palCmdBuffer.h" #include "palCmdAllocator.h" #include "palQueue.h" +#include "palFence.h" #include "palLinearAllocator.h" /*! \addtogroup PAL PAL Resource Implementation @@ -44,7 +46,7 @@ public: static const uint MaxCommands = 512; static const uint StartCmdBufIdx = 1; static const uint FirstMemoryReference = 0x80000000; - static const uint64_t WaitTimeoutInNsec = 60000000000; + static const uint64_t WaitTimeoutInNsec = 6000000000; static Queue* Create( Pal::IDevice* palDev, //!< PAL device object @@ -82,6 +84,30 @@ public: iDev_->RemoveGpuMemoryReferences(1, &iMem, NULL); } + bool waifForFence(uint cbId) const + { + Pal::Result result = Pal::Result::Success; + while (Pal::Result::Success != (result = iCmdFences_[cbId]->GetStatus())) { + if (result == Pal::Result::ErrorFenceNeverSubmitted) { + result = Pal::Result::Success; + break; + } + result = iDev_->WaitForFences(1, &iCmdFences_[cbId], true, WaitTimeoutInNsec); + if (Pal::Result::Success == result) { + break; + } + else if ((Pal::Result::NotReady == result) || + (Pal::Result::Timeout == result)) { + LogWarning("PAL fence isn't ready!"); + } + else { + LogError("PAL wait for a fence failed!"); + break; + } + } + return (result == Pal::Result::Success) ? true : false; + } + //! Flushes the current command buffer to HW //! Returns ID associated with the submission uint submit(bool forceFlush);