diff --git a/rocclr/runtime/device/pal/palconstbuf.cpp b/rocclr/runtime/device/pal/palconstbuf.cpp index f23a860096..bdac07fa4a 100644 --- a/rocclr/runtime/device/pal/palconstbuf.cpp +++ b/rocclr/runtime/device/pal/palconstbuf.cpp @@ -9,64 +9,75 @@ namespace pal { -ConstBuffer::ConstBuffer(VirtualGPU& gpu, size_t size) - : Memory(const_cast(gpu.dev()), size * VectorSize), - gpu_(gpu), - size_(size * VectorSize), - wrtOffset_(0), - lastWrtSize_(0), - wrtAddress_(nullptr) {} +// ================================================================================================ +ManagedBuffer::ManagedBuffer(VirtualGPU& gpu, uint32_t size) + : gpu_(gpu) + , buffers_(MaxNumberOfBuffers) + , activeBuffer_(0) + , size_(size) + , wrtOffset_(0) + , lastWrtSize_(0) + , wrtAddress_(nullptr) {} -ConstBuffer::~ConstBuffer() { - if (wrtAddress_ != nullptr) { - unmap(&gpu_); +// ================================================================================================ +ManagedBuffer::~ManagedBuffer() { + for (auto it : buffers_) { + if (it->data() != nullptr) { + it->unmap(&gpu_); + } + delete it; } - amd::AlignedMemory::deallocate(sysMemCopy_); } -bool ConstBuffer::create() { - // Create sysmem copy for the constant buffer - sysMemCopy_ = reinterpret_cast
(amd::AlignedMemory::allocate(size_, 256)); - if (sysMemCopy_ == nullptr) { - LogPrintfError( - "We couldn't allocate sysmem copy for constant buffer,\ - size(%d)!", - size_); - return false; - } - memset(sysMemCopy_, 0, size_); - - if (!Memory::create(Resource::RemoteUSWC)) { - LogPrintfError("We couldn't create HW constant buffer, size(%d)!", size_); - return false; +// ================================================================================================ +bool ManagedBuffer::create(Resource::MemoryType type, bool constbuf) { + if (constbuf) { + // Create sysmem copy for the constant buffer + sysMemCopy_ = reinterpret_cast
(amd::AlignedMemory::allocate( + gpu_.dev().info().maxParameterSize_, 256)); + if (sysMemCopy_ == nullptr) { + LogPrintfError("We couldn't allocate sysmem copy for constant buffer, size(%d)!", size_); + return false; + } + memset(sysMemCopy_, 0, gpu_.dev().info().maxParameterSize_); } - // Constant buffer warm-up - warmUpRenames(gpu_); - - wrtAddress_ = map(&gpu_, Resource::Discard); - if (wrtAddress_ == nullptr) { - LogPrintfError("We couldn't map HW constant buffer, size(%d)!", size_); - return false; + for (uint i = 0; i < buffers_.size(); ++i) { + buffers_[i] = new Memory(const_cast(gpu_.dev()), size_); + if (nullptr == buffers_[i] || !buffers_[i]->create(type)) { + LogPrintfError("We couldn't create HW constant buffer, size(%d)!", size_); + return false; + } + void* wrtAddress = buffers_[i]->map(&gpu_); + if (wrtAddress == nullptr) { + LogPrintfError("We couldn't map HW constant buffer, size(%d)!", size_); + return false; + } + // Make sure OCL touches every buffer in the queue to avoid delays on the first submit + uint dummy = 0; + static constexpr bool Wait = true; + // Write 0 for the buffer paging by VidMM + buffers_[i]->writeRawData(gpu_, 0, sizeof(dummy), &dummy, Wait); } - + wrtAddress_ = buffers_[activeBuffer_]->data(); return true; } -bool ConstBuffer::uploadDataToHw(size_t size) { - static const size_t HwCbAlignment = 256; +// ================================================================================================ +bool ManagedBuffer::uploadDataToHw(uint32_t size) { + static constexpr uint32_t HwCbAlignment = 256; // Align copy size on the vector's boundary - size_t count = amd::alignUp(size, VectorSize); + uint32_t count = amd::alignUp(size, 16); wrtOffset_ += lastWrtSize_; // Check if CB has enough space for copy if ((wrtOffset_ + count) > size_) { - if (wrtAddress_ != nullptr) { - unmap(&gpu_); - } - wrtAddress_ = map(&gpu_, Resource::Discard); + activeBuffer_ = (++activeBuffer_) % MaxNumberOfBuffers; + //! Make sure buffer isn't busy + buffers_[activeBuffer_]->wait(gpu_); + wrtAddress_ = buffers_[activeBuffer_]->data(); wrtOffset_ = 0; lastWrtSize_ = 0; } diff --git a/rocclr/runtime/device/pal/palconstbuf.hpp b/rocclr/runtime/device/pal/palconstbuf.hpp index 9847e719d4..f227c4eb39 100644 --- a/rocclr/runtime/device/pal/palconstbuf.hpp +++ b/rocclr/runtime/device/pal/palconstbuf.hpp @@ -8,55 +8,61 @@ //! \namespace pal PAL Resource Implementation namespace pal { -//! Cconstant buffer -class ConstBuffer : public Memory { +//! Managed buffer (staging or constant) +class ManagedBuffer : public amd::HeapObject { public: - //! Vector size of the constant buffer - static const size_t VectorSize = 16; - //! Constructor for the ConstBuffer class - ConstBuffer(VirtualGPU& gpu, //!< Virtual GPU device object - size_t size //!< size of the constant buffer in vectors - ); + ManagedBuffer(VirtualGPU& gpu, //!< Virtual GPU device object + uint32_t size //!< size of the managed buffers in bytes + ); //! Destructor for the ConstBuffer class - ~ConstBuffer(); + ~ManagedBuffer(); //! Creates the real HW constant buffer - bool create(); + bool create(Resource::MemoryType type, bool constBuf = false); /*! \brief Uploads current constant buffer data from sysMemCopy_ to HW * * \return True if the data upload was succesful */ - bool uploadDataToHw(size_t size //!< real data size for upload + bool uploadDataToHw(uint32_t size //!< real data size for upload ); //! Returns a pointer to the system memory copy for CB address sysMemCopy() const { return sysMemCopy_; } //! Returns CB size - size_t size() const { return size_; } + uint32_t size() const { return size_; } //! Returns current write offset for the constant buffer - size_t wrtOffset() const { return wrtOffset_; } + uint32_t wrtOffset() const { return wrtOffset_; } //! Returns last write size for the constant buffer - size_t lastWrtSize() const { return lastWrtSize_; } + uint32_t lastWrtSize() const { return lastWrtSize_; } + + Memory* activeMemory() const { return buffers_[activeBuffer_]; } + + uint64_t vmAddress() const { return buffers_[activeBuffer_]->vmAddress(); } private: + //! The maximum number of the managed buffers + static constexpr uint32_t MaxNumberOfBuffers = 3; + //! Disable copy constructor - ConstBuffer(const ConstBuffer&); + ManagedBuffer(const ManagedBuffer&) = delete; //! Disable operator= - ConstBuffer& operator=(const ConstBuffer&); + ManagedBuffer& operator=(const ManagedBuffer&) = delete; - VirtualGPU& gpu_; //!< Virtual GPU object - address sysMemCopy_; //!< System memory copy - size_t size_; //!< Constant buffer size - size_t wrtOffset_; //!< Current write offset - size_t lastWrtSize_; //!< Last write size - void* wrtAddress_; //!< Write address in CB + VirtualGPU& gpu_; //!< Virtual GPU object + std::vector buffers_; //!< Buffers for management + uint32_t activeBuffer_; //!< Current active buffer + address sysMemCopy_; //!< System memory copy + uint32_t size_; //!< Constant buffer size + uint32_t wrtOffset_; //!< Current write offset + uint32_t lastWrtSize_; //!< Last write size + void* wrtAddress_; //!< Write address in CB }; /*@}*/} // namespace pal diff --git a/rocclr/runtime/device/pal/palkernel.cpp b/rocclr/runtime/device/pal/palkernel.cpp index b7e9302cd7..05c22fe6f6 100644 --- a/rocclr/runtime/device/pal/palkernel.cpp +++ b/rocclr/runtime/device/pal/palkernel.cpp @@ -908,10 +908,10 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( AmdAqlWrap* wrap = reinterpret_cast(aqlStruct); memset(wrap, 0, sizeof(AmdAqlWrap)); wrap->state = AQL_WRAP_BUSY; - ConstBuffer* cb = gpu.constBufs_[1]; + ManagedBuffer* cb = gpu.constBufs_[1]; cb->uploadDataToHw(sizeof(AmdAqlWrap)); *vmParentWrap = cb->vmAddress() + cb->wrtOffset(); - gpu.addVmMemory(cb); + gpu.addVmMemory(cb->activeMemory()); } const amd::KernelSignature& signature = kernel.signature(); @@ -1045,12 +1045,12 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( case HSAIL_ARGTYPE_REFERENCE: { // Copy the current structure into CB1 memcpy(aqlStruct, paramaddr, arg->size_); - ConstBuffer* cb = gpu.constBufs_[1]; + ManagedBuffer* cb = gpu.constBufs_[1]; cb->uploadDataToHw(arg->size_); // Then use a pointer in aqlArgBuffer to CB1 size_t gpuPtr = static_cast(cb->vmAddress() + cb->wrtOffset()); WriteAqlArg(&aqlArgBuf, &gpuPtr, sizeof(size_t)); - gpu.addVmMemory(cb); + gpu.addVmMemory(cb->activeMemory()); break; } case HSAIL_ARGTYPE_VALUE: @@ -1080,12 +1080,12 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( if (image->memoryType() == Resource::ImageView) { // Copy the current structre into CB1 memcpy(aqlStruct, image->hwState(), HsaImageObjectSize); - ConstBuffer* cb = gpu.constBufs_[1]; + ManagedBuffer* cb = gpu.constBufs_[1]; cb->uploadDataToHw(HsaImageObjectSize); // Then use a pointer in aqlArgBuffer to CB1 uint64_t srd = cb->vmAddress() + cb->wrtOffset(); WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd)); - gpu.addVmMemory(cb); + gpu.addVmMemory(cb->activeMemory()); } else { uint64_t srd = image->hwSrd(); WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd)); @@ -1175,7 +1175,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( hsaDisp->group_segment_size = ldsAddress - ldsSize(); hsaDisp->kernel_object = gpuAqlCode(); - ConstBuffer* cb = gpu.constBufs_[0]; + ManagedBuffer* cb = gpu.constBufs_[0]; cb->uploadDataToHw(argsBufferSize() + sizeof(hsa_kernel_dispatch_packet_t)); uint64_t argList = cb->vmAddress() + cb->wrtOffset(); @@ -1183,7 +1183,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( hsaDisp->reserved2 = 0; hsaDisp->completion_signal.handle = 0; - gpu.addVmMemory(cb); + gpu.addVmMemory(cb->activeMemory()); gpu.addVmMemory(&prog().codeSegGpu()); for (pal::Memory* mem : prog().globalStores()) { gpu.addVmMemory(mem); diff --git a/rocclr/runtime/device/pal/palvirtual.cpp b/rocclr/runtime/device/pal/palvirtual.cpp index 85dccb0d00..9e468adcf7 100644 --- a/rocclr/runtime/device/pal/palvirtual.cpp +++ b/rocclr/runtime/device/pal/palvirtual.cpp @@ -2803,18 +2803,15 @@ void VirtualGPU::waitEventLock(CommandBatch* cb) { } bool VirtualGPU::allocConstantBuffers() { - // Allocate/reallocate constant buffers - size_t minCbSize; - // GCN doesn't really have a limit - minCbSize = 256 * Ki; + // Allocate constant buffers, GCN doesn't really have a limit + static constexpr uint32_t MinCbSize = 256 * Ki; uint i; // Create/reallocate constant buffer resources for (i = 0; i < MaxConstBuffersArguments; ++i) { - ConstBuffer* constBuf = new ConstBuffer( - *this, ((minCbSize + ConstBuffer::VectorSize - 1) / ConstBuffer::VectorSize)); + ManagedBuffer* constBuf = new ManagedBuffer(*this, MinCbSize); - if ((constBuf != nullptr) && constBuf->create()) { + if ((constBuf != nullptr) && constBuf->create(Resource::RemoteUSWC, true)) { addConstBuffer(constBuf); } else { // We failed to create a constant buffer diff --git a/rocclr/runtime/device/pal/palvirtual.hpp b/rocclr/runtime/device/pal/palvirtual.hpp index fff4332f8b..adcd61fbd7 100644 --- a/rocclr/runtime/device/pal/palvirtual.hpp +++ b/rocclr/runtime/device/pal/palvirtual.hpp @@ -207,7 +207,7 @@ class VirtualGPU : public device::VirtualDevice { State() : value_(0) {} }; - typedef std::vector constbufs_t; + typedef std::vector constbufs_t; class MemoryDependency : public amd::EmbeddedObject { public: @@ -341,10 +341,10 @@ class VirtualGPU : public device::VirtualDevice { ); //! Returns a resource associated with the constant buffer - const ConstBuffer* cb(uint idx) const { return constBufs_[idx]; } + const ManagedBuffer* cb(uint idx) const { return constBufs_[idx]; } //! Adds CAL objects into the constant buffer vector - void addConstBuffer(ConstBuffer* cb) { constBufs_.push_back(cb); } + void addConstBuffer(ManagedBuffer* cb) { constBufs_.push_back(cb); } constbufs_t constBufs_; //!< constant buffers