diff --git a/rocclr/runtime/device/pal/paldevice.cpp b/rocclr/runtime/device/pal/paldevice.cpp index 31ceb453db..1500fdb708 100644 --- a/rocclr/runtime/device/pal/paldevice.cpp +++ b/rocclr/runtime/device/pal/paldevice.cpp @@ -1994,8 +1994,9 @@ bool Device::allocScratch(uint regNum, const VirtualGPU* vgpu, uint vgprs) { regNum = threadSizeLimit; } - // The algorithm below attempts to keep max possible size to allow concurrent execution, - // where the scratch offset will be kept constant - wave_slot * COMPUTE_TMPRING_SIZE.WAVESIZE + // The algorithm below finds the most optimal size for the current execution. + // PAL reprograms COMPUTE_TMPRING_SIZE.WAVESIZE and COMPUTE_TMPRING_SIZE.WAVES on + // every dispatch and sync mode is enabled in runtime // Calculate the size of the scratch buffer for a queue uint32_t numTotalCUs = properties().gfxipProperties.shaderCore.numAvailableCus; @@ -2005,12 +2006,8 @@ bool Device::allocScratch(uint regNum, const VirtualGPU* vgpu, uint vgprs) { numMaxWaves *= properties().gfxipProperties.shaderCore.numSimdsPerCu; // Find max waves per device numMaxWaves = std::min(settings().numScratchWavesPerCu_, numMaxWaves); - // Find max between current alloc and the new limit - numMaxWaves = std::max(numMaxWaves, scratch_[sb]->numMaxWaves_); // Current private mem size uint32_t privateMemSize = regNum * sizeof(uint32_t); - // Max between the allocation and current - privateMemSize = std::max(privateMemSize, scratch_[sb]->privateMemSize_); uint64_t newSize = static_cast(info().wavefrontWidth_) * privateMemSize * numMaxWaves * numTotalCUs; @@ -2035,8 +2032,6 @@ bool Device::allocScratch(uint regNum, const VirtualGPU* vgpu, uint vgprs) { scratchBuf->size_ = std::min(newSize, uint64_t(3 * Gi)); // Note: Generic address space setup in HW requires 64KB alignment for scratch scratchBuf->size_ = amd::alignUp(newSize, 64 * Ki); - scratchBuf->privateMemSize_ = privateMemSize; - scratchBuf->numMaxWaves_ = numMaxWaves; } scratchBuf->offset_ = offset; size += scratchBuf->size_; diff --git a/rocclr/runtime/device/pal/paldevice.hpp b/rocclr/runtime/device/pal/paldevice.hpp index dfb95b0433..0ba85217d3 100644 --- a/rocclr/runtime/device/pal/paldevice.hpp +++ b/rocclr/runtime/device/pal/paldevice.hpp @@ -250,11 +250,9 @@ class Device : public NullDevice { Memory* memObj_; //!< Memory objects for scratch buffers uint64_t offset_; //!< Offset from the global scratch store uint64_t size_; //!< Scratch buffer size on this queue - uint32_t privateMemSize_; //!< Private memory size per thread, allowed by the current scratch - uint32_t numMaxWaves_; //!< The max number of waves for this scratch alloc //! Default constructor - ScratchBuffer() : memObj_(nullptr), offset_(0), size_(0), privateMemSize_(0), numMaxWaves_(0) {} + ScratchBuffer() : memObj_(nullptr), offset_(0), size_(0) {} //! Default constructor ~ScratchBuffer(); diff --git a/rocclr/runtime/device/pal/palvirtual.cpp b/rocclr/runtime/device/pal/palvirtual.cpp index b4d0f351ff..bdc94e1c68 100644 --- a/rocclr/runtime/device/pal/palvirtual.cpp +++ b/rocclr/runtime/device/pal/palvirtual.cpp @@ -2426,24 +2426,16 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const return false; } - const Device::ScratchBuffer* scratch = nullptr; - // Check if the device allocated more registers than the old setup - if (hsaKernel.workGroupInfo()->scratchRegs_ > 0) { - scratch = dev().scratch(hwRing()); - addVmMemory(scratch->memObj_); - } - // Set up the dispatch information Pal::DispatchAqlParams dispatchParam = {}; dispatchParam.pAqlPacket = aqlPkt; - if (nullptr != scratch) { + if (hsaKernel.workGroupInfo()->scratchRegs_ > 0) { + const Device::ScratchBuffer* scratch = nullptr; + scratch = dev().scratch(hwRing()); dispatchParam.scratchAddr = scratch->memObj_->vmAddress(); dispatchParam.scratchSize = scratch->size_; dispatchParam.scratchOffset = scratch->offset_; - // Use maximum available slots for all dispatches to allow async on the same queue - // HW value loaded into SGPR is an offset value calculated as - // wave_slot * COMPUTE_TMPRING_SIZE.WAVESIZE - dispatchParam.workitemPrivateSegmentSize = std::max(hsaKernel.spillSegSize(), scratch->privateMemSize_); + dispatchParam.workitemPrivateSegmentSize = hsaKernel.spillSegSize(); } dispatchParam.pCpuAqlCode = hsaKernel.cpuAqlCode(); dispatchParam.hsaQueueVa = hsaQueueMem_->vmAddress(); @@ -3462,6 +3454,14 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p } addVmMemory(&hsaKernel.prog().codeSegGpu()); + if (hsaKernel.workGroupInfo()->scratchRegs_ > 0) { + const Device::ScratchBuffer* scratch = dev().scratch(hwRing()); + // Validate scratch buffer to force sync mode, because + // the current scratch logic is optimized for size and performance + memoryDependency().validate(*this, scratch->memObj_, IsReadOnly); + addVmMemory(scratch->memObj_); + } + return true; }