From 73b44b613b83172e061eab027d93b4a862883111 Mon Sep 17 00:00:00 2001 From: foreman Date: Fri, 23 Mar 2018 18:41:23 -0400 Subject: [PATCH] P4 to Git Change 1531829 by gandryey@gera-w8 on 2018/03/23 18:34:36 SWDEV-79445 - OCL generic changes and code clean-up - Implement ConstantBuffer management through a single managed buffer. It reduces the amount of allocated buffers on GPU Affected files ... ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palconstbuf.cpp#6 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palconstbuf.hpp#5 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palkernel.cpp#48 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.cpp#79 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palvirtual.hpp#44 edit --- rocclr/runtime/device/pal/palconstbuf.cpp | 71 ++++++++++++++--------- rocclr/runtime/device/pal/palconstbuf.hpp | 58 +++++++++++++----- rocclr/runtime/device/pal/palkernel.cpp | 39 ++++++------- rocclr/runtime/device/pal/palvirtual.cpp | 19 ++++-- rocclr/runtime/device/pal/palvirtual.hpp | 11 ++-- 5 files changed, 129 insertions(+), 69 deletions(-) diff --git a/rocclr/runtime/device/pal/palconstbuf.cpp b/rocclr/runtime/device/pal/palconstbuf.cpp index 5257ce3477..fe4b265fe7 100644 --- a/rocclr/runtime/device/pal/palconstbuf.cpp +++ b/rocclr/runtime/device/pal/palconstbuf.cpp @@ -16,7 +16,6 @@ ManagedBuffer::ManagedBuffer(VirtualGPU& gpu, uint32_t size) , activeBuffer_(0) , size_(size) , wrtOffset_(0) - , lastWrtSize_(0) , wrtAddress_(nullptr) {} // ================================================================================================ @@ -27,23 +26,10 @@ ManagedBuffer::~ManagedBuffer() { } delete it; } - amd::AlignedMemory::deallocate(sysMemCopy_); } // ================================================================================================ -bool ManagedBuffer::create(Resource::MemoryType type, bool constbuf) { - if (constbuf) { - // Create sysmem copy for the constant buffer. - // Allocate extra memory to account the internal arguments - sysMemCopy_ = reinterpret_cast
(amd::AlignedMemory::allocate( - 2 * gpu_.dev().info().maxParameterSize_, 256)); - if (sysMemCopy_ == nullptr) { - LogPrintfError("We couldn't allocate sysmem copy for constant buffer, size(%d)!", size_); - return false; - } - memset(sysMemCopy_, 0, gpu_.dev().info().maxParameterSize_); - } - +bool ManagedBuffer::create(Resource::MemoryType type) { for (uint i = 0; i < buffers_.size(); ++i) { buffers_[i] = new Memory(const_cast(gpu_.dev()), size_); if (nullptr == buffers_[i] || !buffers_[i]->create(type)) { @@ -66,14 +52,13 @@ bool ManagedBuffer::create(Resource::MemoryType type, bool constbuf) { } // ================================================================================================ -bool ManagedBuffer::uploadDataToHw(uint32_t size) { - static constexpr uint32_t HwCbAlignment = 256; +address ManagedBuffer::reserve(uint32_t size, uint64_t* gpu_address) { + static constexpr uint32_t MemAlignment = 256; - // Align copy size on the vector's boundary - uint32_t count = amd::alignUp(size, 16); - wrtOffset_ += lastWrtSize_; + // Align reserve size on the vector's boundary + uint32_t count = amd::alignUp(size, MemAlignment); - // Check if CB has enough space for copy + // Check if buffer has enough space for reservation if ((wrtOffset_ + count) > size_) { // Get the next buffer in the list ++activeBuffer_; @@ -82,15 +67,49 @@ bool ManagedBuffer::uploadDataToHw(uint32_t size) { buffers_[activeBuffer_]->wait(gpu_); wrtAddress_ = buffers_[activeBuffer_]->data(); wrtOffset_ = 0; - lastWrtSize_ = 0; } - // Update memory with new CB data - memcpy((reinterpret_cast(wrtAddress_) + wrtOffset_), sysMemCopy_, count); + *gpu_address = buffers_[activeBuffer_]->vmAddress() + wrtOffset_; + address cpu_address = wrtAddress_ + wrtOffset_; + + // Adjust the offset by the reserved size + wrtOffset_ += count; - // Adjust the size by the HW CB buffer alignment - lastWrtSize_ = amd::alignUp(size, HwCbAlignment); + return cpu_address; +} + +// ================================================================================================ +ConstantBuffer::ConstantBuffer(ManagedBuffer& mbuf, uint32_t size) + : mbuf_(mbuf) + , sys_mem_copy_(nullptr) + , size_(size) +{} + +// ================================================================================================ +ConstantBuffer::~ConstantBuffer() { + amd::AlignedMemory::deallocate(sys_mem_copy_); +} + +// ================================================================================================ +bool ConstantBuffer::Create() { + // Create sysmem copy for the constant buffer. + sys_mem_copy_ = reinterpret_cast
(amd::AlignedMemory::allocate(size_, 256)); + if (sys_mem_copy_ == nullptr) { + LogPrintfError("We couldn't allocate sysmem copy for constant buffer, size(%d)!", size_); + return false; + } + memset(sys_mem_copy_, 0, size_); return true; } +// ================================================================================================ +uint64_t ConstantBuffer::UploadDataToHw(uint32_t size) const +{ + uint64_t vm_address; + address cpu_address = mbuf_.reserve(size, &vm_address); + // Update memory with new CB data + memcpy(cpu_address, sys_mem_copy_, size); + return vm_address; +} + } // namespace pal diff --git a/rocclr/runtime/device/pal/palconstbuf.hpp b/rocclr/runtime/device/pal/palconstbuf.hpp index f227c4eb39..d2595036a1 100644 --- a/rocclr/runtime/device/pal/palconstbuf.hpp +++ b/rocclr/runtime/device/pal/palconstbuf.hpp @@ -20,27 +20,22 @@ class ManagedBuffer : public amd::HeapObject { ~ManagedBuffer(); //! Creates the real HW constant buffer - bool create(Resource::MemoryType type, bool constBuf = false); + bool create(Resource::MemoryType type); /*! \brief Uploads current constant buffer data from sysMemCopy_ to HW * * \return True if the data upload was succesful */ - bool uploadDataToHw(uint32_t size //!< real data size for upload - ); - - //! Returns a pointer to the system memory copy for CB - address sysMemCopy() const { return sysMemCopy_; } + address reserve(uint32_t size, //!< real data size for upload + uint64_t* gpu_address); //! Returns CB size uint32_t size() const { return size_; } - //! Returns current write offset for the constant buffer + //! Returns current write offset for the managed buffer uint32_t wrtOffset() const { return wrtOffset_; } - //! Returns last write size for the constant buffer - uint32_t lastWrtSize() const { return lastWrtSize_; } - + //! Returns active GPU buffer Memory* activeMemory() const { return buffers_[activeBuffer_]; } uint64_t vmAddress() const { return buffers_[activeBuffer_]->vmAddress(); } @@ -58,11 +53,48 @@ class ManagedBuffer : public amd::HeapObject { VirtualGPU& gpu_; //!< Virtual GPU object std::vector buffers_; //!< Buffers for management uint32_t activeBuffer_; //!< Current active buffer - address sysMemCopy_; //!< System memory copy uint32_t size_; //!< Constant buffer size uint32_t wrtOffset_; //!< Current write offset - uint32_t lastWrtSize_; //!< Last write size - void* wrtAddress_; //!< Write address in CB + address wrtAddress_; //!< Write address in CB +}; + +//! Constant buffer +class ConstantBuffer : public amd::HeapObject { +public: + //! Constructor for the ConstBuffer class + ConstantBuffer(ManagedBuffer& mbuf, //!< Managed buffer + uint32_t size + ); + + //! Destructor for the ConstBuffer class + ~ConstantBuffer(); + + //! Creates the real HW constant buffer + bool Create(); + + /*! \brief Uploads current constant buffer data from sysMemCopy_ to HW + * + * \return GPU address for the uploaded data + */ + uint64_t UploadDataToHw(uint32_t size //!< real data size for upload + ) const; + + //! Returns a pointer to the system memory copy for CB + address SysMemCopy(uint32_t size = 0) const { return sys_mem_copy_; } + + //! Returns active GPU buffer + Memory* ActiveMemory() const { return mbuf_.activeMemory(); } + +private: + //! Disable copy constructor + ConstantBuffer(const ConstantBuffer&) = delete; + + //! Disable operator= + ConstantBuffer& operator=(const ConstantBuffer&) = delete; + + ManagedBuffer& mbuf_; //!< Managed buffer on GPU + address sys_mem_copy_; //!< System memory copy + uint32_t size_; //!< Constant buffer size }; /*@}*/} // namespace pal diff --git a/rocclr/runtime/device/pal/palkernel.cpp b/rocclr/runtime/device/pal/palkernel.cpp index 5efba941ff..d8a290ffba 100644 --- a/rocclr/runtime/device/pal/palkernel.cpp +++ b/rocclr/runtime/device/pal/palkernel.cpp @@ -920,8 +920,8 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( const_address parameters, bool nativeMem, uint64_t vmDefQueue, uint64_t* vmParentWrap) const { static const bool WaitOnBusyEngine = true; uint64_t ldsAddress = ldsSize(); - address aqlArgBuf = gpu.cb(0)->sysMemCopy(); - address aqlStruct = gpu.cb(1)->sysMemCopy(); + address aqlArgBuf = gpu.cb(0)->SysMemCopy(); + address aqlStruct = gpu.cb(1)->SysMemCopy(); bool srdResource = false; if (dynamicParallelism()) { @@ -929,10 +929,9 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( AmdAqlWrap* wrap = reinterpret_cast(aqlStruct); memset(wrap, 0, sizeof(AmdAqlWrap)); wrap->state = AQL_WRAP_BUSY; - ManagedBuffer* cb = gpu.constBufs_[1]; - cb->uploadDataToHw(sizeof(AmdAqlWrap)); - *vmParentWrap = cb->vmAddress() + cb->wrtOffset(); - gpu.addVmMemory(cb->activeMemory()); + const ConstantBuffer* cb = gpu.cb(1); + *vmParentWrap = cb->UploadDataToHw(sizeof(AmdAqlWrap)); + gpu.addVmMemory(cb->ActiveMemory()); } const amd::KernelSignature& signature = kernel.signature(); @@ -1066,12 +1065,11 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( case HSAIL_ARGTYPE_REFERENCE: { // Copy the current structure into CB1 memcpy(aqlStruct, paramaddr, arg->size_); - ManagedBuffer* cb = gpu.constBufs_[1]; - cb->uploadDataToHw(arg->size_); + const ConstantBuffer* cb = gpu.cb(1); // Then use a pointer in aqlArgBuffer to CB1 - size_t gpuPtr = static_cast(cb->vmAddress() + cb->wrtOffset()); + size_t gpuPtr = static_cast(cb->UploadDataToHw(arg->size_)); WriteAqlArg(&aqlArgBuf, &gpuPtr, sizeof(size_t), sizeof(size_t)); - gpu.addVmMemory(cb->activeMemory()); + gpu.addVmMemory(cb->ActiveMemory()); break; } case HSAIL_ARGTYPE_VALUE: @@ -1109,12 +1107,11 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( if (image->memoryType() == Resource::ImageView) { // Copy the current structre into CB1 memcpy(aqlStruct, image->hwState(), HsaImageObjectSize); - ManagedBuffer* cb = gpu.constBufs_[1]; - cb->uploadDataToHw(HsaImageObjectSize); + const ConstantBuffer* cb = gpu.cb(1); // Then use a pointer in aqlArgBuffer to CB1 - uint64_t srd = cb->vmAddress() + cb->wrtOffset(); + uint64_t srd = cb->UploadDataToHw(HsaImageObjectSize); WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd), sizeof(srd)); - gpu.addVmMemory(cb->activeMemory()); + gpu.addVmMemory(cb->ActiveMemory()); } else { uint64_t srd = image->hwSrd(); WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd), sizeof(srd)); @@ -1171,15 +1168,15 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( #if defined(WITH_LIGHTNING_COMPILER) // Check there is no arguments' buffer overflow. We may not use all the // hidden argument slots. - assert(aqlArgBuf <= (gpu.cb(0)->sysMemCopy() + argsBufferSize())); + assert(aqlArgBuf <= (gpu.cb(0)->SysMemCopy() + argsBufferSize())); #else // !defined(WITH_LIGHTNING_COMPILER) // HSAIL kernarg segment size is rounded up to multiple of 16. aqlArgBuf = amd::alignUp(aqlArgBuf, 16); - assert((aqlArgBuf == (gpu.cb(0)->sysMemCopy() + argsBufferSize())) && + assert((aqlArgBuf == (gpu.cb(0)->SysMemCopy() + argsBufferSize())) && "Size and the number of arguments don't match!"); #endif // !defined(WITH_LIGHTNING_COMPILER) hsa_kernel_dispatch_packet_t* hsaDisp = - reinterpret_cast(gpu.cb(0)->sysMemCopy() + argsBufferSize()); + reinterpret_cast(gpu.cb(0)->SysMemCopy() + argsBufferSize()); amd::NDRange local(sizes.local()); const amd::NDRange& global = sizes.global(); @@ -1204,15 +1201,15 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( hsaDisp->group_segment_size = ldsAddress - ldsSize(); hsaDisp->kernel_object = gpuAqlCode(); - ManagedBuffer* cb = gpu.constBufs_[0]; - cb->uploadDataToHw(argsBufferSize() + sizeof(hsa_kernel_dispatch_packet_t)); - uint64_t argList = cb->vmAddress() + cb->wrtOffset(); + const ConstantBuffer* cb = gpu.cb(0); + uint64_t argList = cb->UploadDataToHw( + argsBufferSize() + sizeof(hsa_kernel_dispatch_packet_t)); hsaDisp->kernarg_address = reinterpret_cast(argList); hsaDisp->reserved2 = 0; hsaDisp->completion_signal.handle = 0; - gpu.addVmMemory(cb->activeMemory()); + gpu.addVmMemory(cb->ActiveMemory()); gpu.addVmMemory(&prog().codeSegGpu()); for (pal::Memory* mem : prog().globalStores()) { gpu.addVmMemory(mem); diff --git a/rocclr/runtime/device/pal/palvirtual.cpp b/rocclr/runtime/device/pal/palvirtual.cpp index 9e468adcf7..fd4e3fa67b 100644 --- a/rocclr/runtime/device/pal/palvirtual.cpp +++ b/rocclr/runtime/device/pal/palvirtual.cpp @@ -702,6 +702,7 @@ VirtualGPU::VirtualGPU(Device& device) printfDbgHSA_(nullptr), tsCache_(nullptr), dmaFlushMgmt_(device), + writeBuffer_(nullptr), hwRing_(0), readjustTimeGPU_(0), lastTS_(nullptr), @@ -812,6 +813,13 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs, Unimplemented(); } + writeBuffer_ = new ManagedBuffer(*this, dev().settings().stagedXferSize_); + if ((writeBuffer_ == nullptr) || !writeBuffer_->create(Resource::RemoteUSWC)) { + // We failed to create a constant buffer + delete writeBuffer_; + return false; + } + // Diable double copy optimization, // since UAV read from nonlocal is fast enough blitSetup.disableCopyBufferToImageOpt_ = true; @@ -934,6 +942,8 @@ VirtualGPU::~VirtualGPU() { delete constBufs_[i]; } + delete writeBuffer_; + //! @todo Temporarily keep the buffer mapped for debug purpose if (nullptr != schedParams_) { schedParams_->unmap(this); @@ -2803,15 +2813,16 @@ void VirtualGPU::waitEventLock(CommandBatch* cb) { } bool VirtualGPU::allocConstantBuffers() { - // Allocate constant buffers, GCN doesn't really have a limit - static constexpr uint32_t MinCbSize = 256 * Ki; + // Allocate constant buffers. + // Use double size, reported to the app to account for internal arguments + const uint32_t MinCbSize = 2 * dev().info().maxParameterSize_; uint i; // Create/reallocate constant buffer resources for (i = 0; i < MaxConstBuffersArguments; ++i) { - ManagedBuffer* constBuf = new ManagedBuffer(*this, MinCbSize); + ConstantBuffer* constBuf = new ConstantBuffer(*writeBuffer_, MinCbSize); - if ((constBuf != nullptr) && constBuf->create(Resource::RemoteUSWC, true)) { + if ((constBuf != nullptr) && constBuf->Create()) { addConstBuffer(constBuf); } else { // We failed to create a constant buffer diff --git a/rocclr/runtime/device/pal/palvirtual.hpp b/rocclr/runtime/device/pal/palvirtual.hpp index adcd61fbd7..5304ccbfd4 100644 --- a/rocclr/runtime/device/pal/palvirtual.hpp +++ b/rocclr/runtime/device/pal/palvirtual.hpp @@ -207,7 +207,7 @@ class VirtualGPU : public device::VirtualDevice { State() : value_(0) {} }; - typedef std::vector constbufs_t; + typedef std::vector constbufs_t; class MemoryDependency : public amd::EmbeddedObject { public: @@ -341,12 +341,10 @@ class VirtualGPU : public device::VirtualDevice { ); //! Returns a resource associated with the constant buffer - const ManagedBuffer* cb(uint idx) const { return constBufs_[idx]; } + const ConstantBuffer* cb(uint idx) const { return constBufs_[idx]; } //! Adds CAL objects into the constant buffer vector - void addConstBuffer(ManagedBuffer* cb) { constBufs_.push_back(cb); } - - constbufs_t constBufs_; //!< constant buffers + void addConstBuffer(ConstantBuffer* cb) { constBufs_.push_back(cb); } //! Start the command profiling void profilingBegin(amd::Command& command, //!< Command queue object @@ -597,6 +595,9 @@ class VirtualGPU : public device::VirtualDevice { std::vector xferWriteBuffers_; //!< Stage write buffers std::vector pinnedMems_; //!< Pinned memory list + ManagedBuffer* writeBuffer_; //!< Managed write buffer + constbufs_t constBufs_; //!< constant buffers + typedef std::queue CommandBatchQueue; CommandBatchQueue cbQueue_; //!< Queue of command batches CommandBatchQueue freeCbQueue_; //!< Queue of free command batches