P4 to Git Change 1989702 by gandryey@gera-win10 on 2019/08/27 11:49:38

SWDEV-200489 - [CQE OCL][QR][Windows][Vega20][19H1] Performance drop is observed while running Blender on Vega20 due to faulty CL#1981122
	- Switch scratch buffer allocation algorithm back to the optimal size calculation with sync mode. Some kernels will run slower if max scratch per queue is programmed unconditionally due to possible lower memory efficiency with fetches

Affected files ...

... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#160 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.hpp#45 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#149 edit

rocclr/runtime/device/pal/paldevice.cpp

@@ -1994,8 +1994,9 @@ bool Device::allocScratch(uint regNum, const VirtualGPU* vgpu, uint vgprs) {
       regNum = threadSizeLimit;
     }
 
-    // The algorithm below attempts to keep max possible size to allow concurrent execution,
-    // where the scratch offset will be kept constant - wave_slot * COMPUTE_TMPRING_SIZE.WAVESIZE
+    // The algorithm below finds the most optimal size for the current execution.
+    // PAL reprograms COMPUTE_TMPRING_SIZE.WAVESIZE and COMPUTE_TMPRING_SIZE.WAVES on
+    // every dispatch and sync mode is enabled in runtime
 
     // Calculate the size of the scratch buffer for a queue
     uint32_t numTotalCUs = properties().gfxipProperties.shaderCore.numAvailableCus;
@@ -2005,12 +2006,8 @@ bool Device::allocScratch(uint regNum, const VirtualGPU* vgpu, uint vgprs) {
     numMaxWaves *= properties().gfxipProperties.shaderCore.numSimdsPerCu;
     // Find max waves per device
     numMaxWaves = std::min(settings().numScratchWavesPerCu_, numMaxWaves);
-    // Find max between current alloc and the new limit
-    numMaxWaves = std::max(numMaxWaves, scratch_[sb]->numMaxWaves_);
     // Current private mem size
     uint32_t privateMemSize = regNum * sizeof(uint32_t);
-    // Max between the allocation and current
-    privateMemSize = std::max(privateMemSize, scratch_[sb]->privateMemSize_);
     uint64_t newSize =
         static_cast<uint64_t>(info().wavefrontWidth_) * privateMemSize * numMaxWaves * numTotalCUs;
 
@@ -2035,8 +2032,6 @@ bool Device::allocScratch(uint regNum, const VirtualGPU* vgpu, uint vgprs) {
             scratchBuf->size_ = std::min(newSize, uint64_t(3 * Gi));
             // Note: Generic address space setup in HW requires 64KB alignment for scratch
             scratchBuf->size_ = amd::alignUp(newSize, 64 * Ki);
-            scratchBuf->privateMemSize_ = privateMemSize;
-            scratchBuf->numMaxWaves_ = numMaxWaves;
           }
           scratchBuf->offset_ = offset;
           size += scratchBuf->size_;

rocclr/runtime/device/pal/paldevice.hpp

@@ -250,11 +250,9 @@ class Device : public NullDevice {
     Memory* memObj_;           //!< Memory objects for scratch buffers
     uint64_t offset_;          //!< Offset from the global scratch store
     uint64_t size_;            //!< Scratch buffer size on this queue
-    uint32_t privateMemSize_;  //!< Private memory size per thread, allowed by the current scratch
-    uint32_t numMaxWaves_;     //!< The max number of waves for this scratch alloc
 
     //! Default constructor
-    ScratchBuffer() : memObj_(nullptr), offset_(0), size_(0), privateMemSize_(0), numMaxWaves_(0) {}
+    ScratchBuffer() : memObj_(nullptr), offset_(0), size_(0) {}
 
     //! Default constructor
     ~ScratchBuffer();

rocclr/runtime/device/pal/palvirtual.cpp

@@ -2426,24 +2426,16 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const
       return false;
     }
 
-    const Device::ScratchBuffer* scratch = nullptr;
-    // Check if the device allocated more registers than the old setup
-    if (hsaKernel.workGroupInfo()->scratchRegs_ > 0) {
-      scratch = dev().scratch(hwRing());
-      addVmMemory(scratch->memObj_);
-    }
-
     // Set up the dispatch information
     Pal::DispatchAqlParams dispatchParam = {};
     dispatchParam.pAqlPacket = aqlPkt;
-    if (nullptr != scratch) {
+    if (hsaKernel.workGroupInfo()->scratchRegs_ > 0) {
+      const Device::ScratchBuffer* scratch = nullptr;
+      scratch = dev().scratch(hwRing());
       dispatchParam.scratchAddr = scratch->memObj_->vmAddress();
       dispatchParam.scratchSize = scratch->size_;
       dispatchParam.scratchOffset = scratch->offset_;
-      // Use maximum available slots for all dispatches to allow async on the same queue
-      // HW value loaded into SGPR is an offset value calculated as
-      // wave_slot * COMPUTE_TMPRING_SIZE.WAVESIZE
-      dispatchParam.workitemPrivateSegmentSize = std::max(hsaKernel.spillSegSize(), scratch->privateMemSize_);
+      dispatchParam.workitemPrivateSegmentSize = hsaKernel.spillSegSize();
     }
     dispatchParam.pCpuAqlCode = hsaKernel.cpuAqlCode();
     dispatchParam.hsaQueueVa = hsaQueueMem_->vmAddress();
@@ -3462,6 +3454,14 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p
   }
   addVmMemory(&hsaKernel.prog().codeSegGpu());
 
+  if (hsaKernel.workGroupInfo()->scratchRegs_ > 0) {
+    const Device::ScratchBuffer* scratch = dev().scratch(hwRing());
+    // Validate scratch buffer to force sync mode, because
+    // the current scratch logic is optimized for size and performance
+    memoryDependency().validate(*this, scratch->memObj_, IsReadOnly);
+    addVmMemory(scratch->memObj_);
+  }
+
   return true;
 }