diff --git a/projects/clr/rocclr/runtime/device/blit.hpp b/projects/clr/rocclr/runtime/device/blit.hpp index 386a979e92..3cf6a283cb 100644 --- a/projects/clr/rocclr/runtime/device/blit.hpp +++ b/projects/clr/rocclr/runtime/device/blit.hpp @@ -177,6 +177,9 @@ class BlitManager : public amd::HeapObject { //! Enables synchronization on blit operations void enableSynchronization() { syncOperation_ = true; } + + //! Returns Xfer queue lock + virtual amd::Monitor* lockXfer() const { return nullptr; } protected: const Setup setup_; //!< HW accelerated blit requested diff --git a/projects/clr/rocclr/runtime/device/pal/palblit.cpp b/projects/clr/rocclr/runtime/device/pal/palblit.cpp index ecf3390fb0..4415ac0f0d 100644 --- a/projects/clr/rocclr/runtime/device/pal/palblit.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palblit.cpp @@ -19,7 +19,6 @@ DmaBlitManager::DmaBlitManager(VirtualGPU& gpu, Setup setup) inline void DmaBlitManager::synchronize() const { if (syncOperation_) { - gpu().releaseMemObjects(); gpu().waitAllEngines(); } } @@ -2311,6 +2310,7 @@ bool KernelBlitManager::runScheduler(device::Memory& vqueue, device::Memory& par } void KernelBlitManager::writeRawData(device::Memory& memory, size_t size, const void* data) const { + amd::ScopedLock k(lockXferOps_); static_cast(memory).writeRawData(gpu(), 0, size, data, false); synchronize(); diff --git a/projects/clr/rocclr/runtime/device/pal/palblit.hpp b/projects/clr/rocclr/runtime/device/pal/palblit.hpp index 4e7e0187a3..ece29da86d 100644 --- a/projects/clr/rocclr/runtime/device/pal/palblit.hpp +++ b/projects/clr/rocclr/runtime/device/pal/palblit.hpp @@ -352,6 +352,8 @@ class KernelBlitManager : public DmaBlitManager { const void* data //!< Raw data pointer ) const; + virtual amd::Monitor* lockXfer() const { return lockXferOps_; } + private: static const size_t MaxXferBuffers = 2; static const uint TransferSplitSize = 3; diff --git a/projects/clr/rocclr/runtime/device/pal/paldefs.hpp b/projects/clr/rocclr/runtime/device/pal/paldefs.hpp index cccc22eee6..449cd9233a 100644 --- a/projects/clr/rocclr/runtime/device/pal/paldefs.hpp +++ b/projects/clr/rocclr/runtime/device/pal/paldefs.hpp @@ -45,11 +45,14 @@ enum EngineType { MainEngine = 0, SdmaEngine, AllEngines }; struct GpuEvent { static const unsigned int InvalidID = ((1 << 30) - 1); - EngineType engineId_; ///< type of the id - unsigned int id; ///< actual event id - + struct { + uint32_t id : 31; ///< actual event id + uint32_t engineId_ : 1; ///< type of the id + }; //! GPU event default constructor GpuEvent() : engineId_(MainEngine), id(InvalidID) {} + //! GPU event constructor + GpuEvent(uint evt) : engineId_(MainEngine), id(evt) {} //! Returns true if the current event is valid bool isValid() const { return (id != InvalidID) ? true : false; } diff --git a/projects/clr/rocclr/runtime/device/pal/paldevice.cpp b/projects/clr/rocclr/runtime/device/pal/paldevice.cpp index 5e2292429f..26a65db582 100644 --- a/projects/clr/rocclr/runtime/device/pal/paldevice.cpp +++ b/projects/clr/rocclr/runtime/device/pal/paldevice.cpp @@ -581,7 +581,10 @@ Device::Device() heapInitComplete_(false), xferQueue_(nullptr), globalScratchBuf_(nullptr), - srdManager_(nullptr) {} + srdManager_(nullptr), + lockResourceOps_(nullptr), + resourceList_(nullptr) + {} Device::~Device() { // remove the HW debug manager @@ -625,6 +628,8 @@ Device::~Device() { delete vgpusAccess_; delete scratchAlloc_; delete mapCacheOps_; + delete lockResourceOps_; + delete resourceList_; if (context_ != nullptr) { context_->release(); @@ -639,7 +644,10 @@ bool Device::create(Pal::IDevice* device) { if (!amd::Device::create()) { return false; } - + resourceList_ = new std::list(); + if (nullptr == resourceList_) { + return false; + } appProfile_.init(); device_ = device; Pal::Result result; @@ -749,6 +757,11 @@ bool Device::create(Pal::IDevice* device) { return false; } + lockResourceOps_ = new amd::Monitor("Resource List Ops Lock", true); + if (nullptr == lockResourceOps_) { + return false; + } + lockForInitHeap_ = new amd::Monitor("Async Ops Lock For Initialization of Heap Resource", true); if (nullptr == lockForInitHeap_) { return false; @@ -2075,9 +2088,9 @@ bool Device::createBlitProgram() { return result; } -void Device::SrdManager::fillResourceList(std::vector& memList) { +void Device::SrdManager::fillResourceList(VirtualGPU& gpu) { for (uint i = 0; i < pool_.size(); ++i) { - memList.push_back(pool_[i].buf_); + gpu.addVmMemory(pool_[i].buf_); } } diff --git a/projects/clr/rocclr/runtime/device/pal/paldevice.hpp b/projects/clr/rocclr/runtime/device/pal/paldevice.hpp index 3c5b2930e8..4e39628616 100644 --- a/projects/clr/rocclr/runtime/device/pal/paldevice.hpp +++ b/projects/clr/rocclr/runtime/device/pal/paldevice.hpp @@ -146,6 +146,7 @@ class Program; class Kernel; class Memory; class Resource; +class GpuMemoryReference; class VirtualDevice; class PrintfDbg; class ThreadTrace; @@ -276,7 +277,7 @@ class Device : public NullDevice { void freeSrdSlot(uint64_t addr); // Fills the memory list for VidMM KMD - void fillResourceList(std::vector& memList); + void fillResourceList(VirtualGPU& gpu); private: //! Disable copy constructor @@ -369,6 +370,9 @@ class Device : public NullDevice { //! Returns the monitor object for PAL amd::Monitor& lockPAL() const { return *lockPAL_; } + //! Returns the monitor object for PAL + amd::Monitor& lockResources() const { return *lockResourceOps_; } + //! Returns the number of virtual GPUs allocated on this device uint numOfVgpus() const { return numOfVgpus_; } uint numOfVgpus_; //!< The number of virtual GPUs (lock protected) @@ -487,6 +491,41 @@ class Device : public NullDevice { bool resGLRelease(void* GLplatformContext, void* mbResHandle, uint type) const; bool resGLFree(void* GLplatformContext, void* mbResHandle, uint type) const; + //! Adds a resource to the global list + void addResource(GpuMemoryReference* mem) const { + amd::ScopedLock lock(lockResources()); + auto findIt = std::find(resourceList_->begin(), resourceList_->end(), mem); + mem->events_.resize(numOfVgpus()); + if (resourceList_->end() == findIt) { + resourceList_->push_back(mem); + } + } + + //! Removes a resource from the global list + void removeResource(GpuMemoryReference* mem) const { + amd::ScopedLock lock(lockResources()); + resourceList_->remove(mem); + } + + //! Resizes global resource list to accumulate a new queue + void resizeResoureList(uint index) const { + // Not safe to resize the list when runtime creates/destroys a queue at the same time + // or other queues process a command, since the size of the TS array can change + Device::ScopedLockVgpus v(*this); + amd::ScopedLock r(lockResources()); + for (auto it : *resourceList_) { + it->resizeGpuEvents(index); + } + } + + //! Erases an old queue from the list + void eraseResoureList(uint index) const { + amd::ScopedLock lock(lockResources()); + for (auto it : *resourceList_) { + it->eraseGpuEvents(index); + } + } + private: //! Disable copy constructor Device(const Device&); @@ -548,6 +587,8 @@ class Device : public NullDevice { Pal::DeviceProperties properties_; //!< PAL device properties Pal::IDevice* device_; //!< PAL device object std::atomic freeMem[Pal::GpuHeap::GpuHeapCount]; //!< Free memory counter + amd::Monitor* lockResourceOps_; //!< Lock to serialise resource access + std::list* resourceList_; //!< Active resource list }; /*@}*/} // namespace pal diff --git a/projects/clr/rocclr/runtime/device/pal/palkernel.cpp b/projects/clr/rocclr/runtime/device/pal/palkernel.cpp index a7a201e5c1..d16aaf89b4 100644 --- a/projects/clr/rocclr/runtime/device/pal/palkernel.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palkernel.cpp @@ -883,8 +883,7 @@ const uint16_t kDispatchPacketHeader = (HSA_PACKET_TYPE_KERNEL_DISPATCH << HSA_P hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( VirtualGPU& gpu, const amd::Kernel& kernel, const amd::NDRangeContainer& sizes, - const_address parameters, bool nativeMem, uint64_t vmDefQueue, uint64_t* vmParentWrap, - std::vector& memList) const { + const_address parameters, bool nativeMem, uint64_t vmDefQueue, uint64_t* vmParentWrap) const { static const bool WaitOnBusyEngine = true; uint64_t ldsAddress = ldsSize(); address aqlArgBuf = gpu.cb(0)->sysMemCopy(); @@ -899,7 +898,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( ConstBuffer* cb = gpu.constBufs_[1]; cb->uploadDataToHw(sizeof(AmdAqlWrap)); *vmParentWrap = cb->vmAddress() + cb->wrtOffset(); - memList.push_back(cb); + gpu.addVmMemory(cb); } const amd::KernelSignature& signature = kernel.signature(); @@ -940,7 +939,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( (gpu.printfDbgHSA().dbgBuffer() != nullptr)) { // and set the fourth argument as the printf_buffer pointer bufferPtr = static_cast(gpu.printfDbgHSA().dbgBuffer()->vmAddress()); - memList.push_back(gpu.printfDbgHSA().dbgBuffer()); + gpu.addVmMemory(gpu.printfDbgHSA().dbgBuffer()); } assert(arg->size_ == sizeof(bufferPtr) && "check the sizes"); WriteAqlArg(&aqlArgBuf, &bufferPtr, arg->size_, arg->alignment_); @@ -985,7 +984,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( if ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0) { mem->signalWrite(&dev()); } - memList.push_back(gpuMem); + gpu.addVmMemory(gpuMem); } // If finegrainsystem is present then the pointer can be malloced by the app and // passed to kernel directly. If so copy the pointer location to aqlArgBuf @@ -1022,7 +1021,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( if ((nullptr != mem) && ((mem->getMemFlags() & CL_MEM_READ_ONLY) == 0)) { mem->signalWrite(&dev()); } - memList.push_back(gpuMem); + gpu.addVmMemory(gpuMem); // save the memory object pointer to allow global memory access if (nullptr != dev().hwDebugMgr()) { @@ -1038,7 +1037,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( // Then use a pointer in aqlArgBuffer to CB1 size_t gpuPtr = static_cast(cb->vmAddress() + cb->wrtOffset()); WriteAqlArg(&aqlArgBuf, &gpuPtr, sizeof(size_t)); - memList.push_back(cb); + gpu.addVmMemory(cb); break; } case HSAIL_ARGTYPE_VALUE: @@ -1073,7 +1072,7 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( // Then use a pointer in aqlArgBuffer to CB1 uint64_t srd = cb->vmAddress() + cb->wrtOffset(); WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd)); - memList.push_back(cb); + gpu.addVmMemory(cb); } else { uint64_t srd = image->hwSrd(); WriteAqlArg(&aqlArgBuf, &srd, sizeof(srd)); @@ -1085,7 +1084,11 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( mem->signalWrite(&dev()); } - memList.push_back(image); + gpu.addVmMemory(image); + if (image->desc().isDoppTexture_) { + gpu.addDoppRef(image, kernel.parameters().getExecNewVcop(), + kernel.parameters().getExecPfpaVcop()); + } break; } case HSAIL_ARGTYPE_SAMPLER: { @@ -1167,18 +1170,18 @@ hsa_kernel_dispatch_packet_t* HSAILKernel::loadArguments( hsaDisp->reserved2 = 0; hsaDisp->completion_signal.handle = 0; - memList.push_back(cb); - memList.push_back(&prog().codeSegGpu()); + gpu.addVmMemory(cb); + gpu.addVmMemory(&prog().codeSegGpu()); for (pal::Memory* mem : prog().globalStores()) { - memList.push_back(mem); + gpu.addVmMemory(mem); } if (AMD_HSA_BITS_GET(cpuAqlCode_->kernel_code_properties, AMD_KERNEL_CODE_PROPERTIES_ENABLE_SGPR_QUEUE_PTR)) { - memList.push_back(gpu.hsaQueueMem()); + gpu.addVmMemory(gpu.hsaQueueMem()); } if (srdResource || prog().isStaticSampler()) { - dev().srds().fillResourceList(memList); + dev().srds().fillResourceList(gpu); } return hsaDisp; diff --git a/projects/clr/rocclr/runtime/device/pal/palkernel.hpp b/projects/clr/rocclr/runtime/device/pal/palkernel.hpp index c059b6d5bb..d73a20969c 100644 --- a/projects/clr/rocclr/runtime/device/pal/palkernel.hpp +++ b/projects/clr/rocclr/runtime/device/pal/palkernel.hpp @@ -187,8 +187,7 @@ class HSAILKernel : public device::Kernel { const_address parameters, //!< Application arguments for the kernel bool nativeMem, //!< Native memory objects are passed uint64_t vmDefQueue, //!< GPU VM default queue pointer - uint64_t* vmParentWrap, //!< GPU VM parent aql wrap object - std::vector& memList //!< Memory list for GSL/VidMM handles + uint64_t* vmParentWrap //!< GPU VM parent aql wrap object ) const; diff --git a/projects/clr/rocclr/runtime/device/pal/palmemory.cpp b/projects/clr/rocclr/runtime/device/pal/palmemory.cpp index b8e017c51d..f417e072fc 100644 --- a/projects/clr/rocclr/runtime/device/pal/palmemory.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palmemory.cpp @@ -169,6 +169,13 @@ bool Memory::create(Resource::MemoryType memType, Resource::CreateParams* params } } + if (result) { + dev().addResource(memRef()); + if (params != nullptr) { + memRef()->gpu_ = params->gpu_; + } + } + return result; } @@ -363,10 +370,6 @@ bool Memory::createInterop(InteropType type) { } oglRes.glPlatformContext_ = owner()->getContext().info().hCtx_; - oglRes.glDeviceContext_ = - owner()->getContext().info().hDev_[amd::Context::DeviceFlagIdx::GLDeviceKhrIdx]; - // We dont pass any flags here for the GL Resource. - oglRes.flags_ = 0; } else { return false; } diff --git a/projects/clr/rocclr/runtime/device/pal/palprogram.cpp b/projects/clr/rocclr/runtime/device/pal/palprogram.cpp index c252af2e74..a2842d4509 100644 --- a/projects/clr/rocclr/runtime/device/pal/palprogram.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palprogram.cpp @@ -86,6 +86,7 @@ void Segment::copy(size_t offset, const void* src, size_t size) { if (cpuAccess_ != nullptr) { amd::Os::fastMemcpy(cpuAddress(offset), src, size); } else { + amd::ScopedLock k(gpuAccess_->dev().xferMgr().lockXfer()); VirtualGPU& gpu = *gpuAccess_->dev().xferQueue(); Memory& xferBuf = gpuAccess_->dev().xferWrite().acquire(); size_t tmpSize = std::min(static_cast(xferBuf.vmSize()), size); @@ -98,7 +99,6 @@ void Segment::copy(size_t offset, const void* src, size_t size) { srcOffs += tmpSize; tmpSize = std::min(static_cast(xferBuf.vmSize()), size); } - gpu.releaseMemObjects(); gpu.waitAllEngines(); } } @@ -108,8 +108,8 @@ bool Segment::freeze(bool destroySysmem) { bool result = true; if (cpuAccess_ != nullptr) { assert(gpuAccess_->size() == cpuAccess_->size() && "Backing store size mismatch!"); + amd::ScopedLock k(gpuAccess_->dev().xferMgr().lockXfer()); result = cpuAccess_->partialMemCopyTo(gpu, 0, 0, gpuAccess_->size(), *gpuAccess_, false, true); - gpu.releaseMemObjects(); gpu.waitAllEngines(); } assert(!destroySysmem || (cpuAccess_ == nullptr)); @@ -813,8 +813,8 @@ bool HSAILProgram::allocKernelTable() { return true; } -void HSAILProgram::fillResListWithKernels(std::vector& memList) const { - memList.push_back(&codeSegGpu()); +void HSAILProgram::fillResListWithKernels(VirtualGPU& gpu) const { + gpu.addVmMemory(&codeSegGpu()); } const aclTargetInfo& HSAILProgram::info(const char* str) { diff --git a/projects/clr/rocclr/runtime/device/pal/palprogram.hpp b/projects/clr/rocclr/runtime/device/pal/palprogram.hpp index 4156bda612..1f57550ef1 100644 --- a/projects/clr/rocclr/runtime/device/pal/palprogram.hpp +++ b/projects/clr/rocclr/runtime/device/pal/palprogram.hpp @@ -150,7 +150,7 @@ class HSAILProgram : public device::Program { const Memory* kernelTable() const { return kernels_; } //! Adds all kernels to the mem handle lists - void fillResListWithKernels(std::vector& memList) const; + void fillResListWithKernels(VirtualGPU& gpu) const; //! Returns the maximum number of scratch regs used in the program uint maxScratchRegs() const { return maxScratchRegs_; } diff --git a/projects/clr/rocclr/runtime/device/pal/palresource.cpp b/projects/clr/rocclr/runtime/device/pal/palresource.cpp index a8758e8959..9c75bd3553 100644 --- a/projects/clr/rocclr/runtime/device/pal/palresource.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palresource.cpp @@ -36,7 +36,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, return nullptr; } - GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(); + GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev); if (memRef != nullptr) { result = dev.iDev()->CreateGpuMemory(createInfo, &memRef[1], &memRef->gpuMem_); if (result != Pal::Result::Success) { @@ -57,7 +57,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, return nullptr; } - GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(); + GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev); Pal::VaRange vaRange = Pal::VaRange::Default; if (memRef != nullptr) { result = dev.iDev()->CreatePinnedGpuMemory(createInfo, &memRef[1], &memRef->gpuMem_); @@ -67,8 +67,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, } } // Update free memory size counters - const_cast(dev).updateFreeMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size, - false); + const_cast(dev).updateFreeMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size, false); return memRef; } @@ -80,7 +79,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, return nullptr; } - GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(); + GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev); if (memRef != nullptr) { result = dev.iDev()->CreateSvmGpuMemory(createInfo, &memRef[1], &memRef->gpuMem_); if (result != Pal::Result::Success) { @@ -103,7 +102,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, } Pal::GpuMemoryCreateInfo createInfo = {}; - GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(); + GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev); if (memRef != nullptr) { result = dev.iDev()->OpenExternalSharedGpuMemory(openInfo, &memRef[1], &createInfo, &memRef->gpuMem_); @@ -112,7 +111,6 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, return nullptr; } } - return memRef; } @@ -129,7 +127,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, } Pal::GpuMemoryCreateInfo createInfo = {}; - GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(); + GpuMemoryReference* memRef = new (gpuMemSize) GpuMemoryReference(dev); char* imgMem = new char[imageSize]; if (memRef != nullptr) { result = dev.iDev()->OpenExternalSharedImage(openInfo, imgMem, &memRef[1], &createInfo, image, @@ -139,20 +137,46 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, return nullptr; } } - return memRef; } -GpuMemoryReference::GpuMemoryReference() : gpuMem_(nullptr), cpuAddress_(nullptr) {} +GpuMemoryReference::GpuMemoryReference(const Device& dev) + : gpuMem_(nullptr), cpuAddress_(nullptr), events_(dev.numOfVgpus()), device_(dev), gpu_(nullptr), resident_(0) {} GpuMemoryReference::~GpuMemoryReference() { - if (cpuAddress_ != nullptr) { - iMem()->Unmap(); + if (gpu_ == nullptr) { + { + Device::ScopedLockVgpus lock(device_); + // Release all memory objects on all virtual GPUs + for (uint idx = 1; idx < device_.vgpus().size(); ++idx) { + device_.vgpus()[idx]->releaseMemory(this, &events_[idx]); + } + } + } else { + gpu_->releaseMemory(this, &events_[gpu_->index()]); } - if (0 != iMem()) { - iMem()->Destroy(); - gpuMem_ = nullptr; + if (device_.vgpus().size() != 0) { + assert(device_.vgpus()[0] == device_.xferQueue() && "Wrong transfer queue!"); + // Lock the transfer queue, since it's not handled by ScopedLockVgpus + amd::ScopedLock k(device_.xferMgr().lockXfer()); + device_.vgpus()[0]->releaseMemory(this, &events_[0]); } + + if (resident_ != 0) { + LogError("Residency counter isn't 0 on memory destroy!"); + } + + { + amd::ScopedLock lk(device_.lockPAL()); + if (cpuAddress_ != nullptr) { + iMem()->Unmap(); + } + if (0 != iMem()) { + iMem()->Destroy(); + gpuMem_ = nullptr; + } + } + device_.removeResource(this); } Resource::Resource(const Device& gpuDev, size_t size) @@ -165,7 +189,6 @@ Resource::Resource(const Device& gpuDev, size_t size) memRef_(nullptr), viewOwner_(nullptr), pinOffset_(0), - gpu_(nullptr), image_(nullptr), hwSrd_(0) { // Fill resource descriptor fields @@ -203,7 +226,6 @@ Resource::Resource(const Device& gpuDev, size_t width, size_t height, size_t dep memRef_(nullptr), viewOwner_(nullptr), pinOffset_(0), - gpu_(nullptr), image_(nullptr), hwSrd_(0) { // Fill resource descriptor fields @@ -428,10 +450,6 @@ bool Resource::create(MemoryType memType, CreateParams* params) { desc_.type_ = RemoteUSWC; } - if (params != nullptr) { - gpu_ = params->gpu_; - } - Pal::Result result; #ifdef _WIN32 @@ -465,7 +483,6 @@ bool Resource::create(MemoryType memType, CreateParams* params) { break; } glPlatformContext_ = oglRes->glPlatformContext_; - glDeviceContext_ = oglRes->glDeviceContext_; layer = oglRes->layer_; type = oglRes->type_; mipLevel = oglRes->mipLevel_; @@ -1055,7 +1072,6 @@ bool Resource::create(MemoryType memType, CreateParams* params) { return false; } } - return true; } @@ -1069,71 +1085,44 @@ void Resource::free() { LogWarning("Resource wasn't unlocked, but destroyed!"); } const bool wait = - (memoryType() != ImageView) && (memoryType() != ImageBuffer) && (memoryType() != View); + (memoryType() != ImageView) && (memoryType() != ImageBuffer) && (memoryType() != View); - // Check if resource could be used in any queue(thread) - if (gpu_ == nullptr) { - Device::ScopedLockVgpus lock(dev()); - - if (renames_.size() == 0) { - // Destroy GSL resource - if (iMem() != 0) { - // Release all virtual memory objects on all virtual GPUs - for (uint idx = 0; idx < dev().vgpus().size(); ++idx) { - // Ignore the transfer queue, - // since it releases resources after every operation - if (dev().vgpus()[idx] != dev().xferQueue()) { - dev().vgpus()[idx]->releaseMemory(iMem(), wait); - } - } - - //! @note: This is a workaround for bad applications that - //! don't unmap memory - if (mapCount_ != 0) { - unmap(nullptr); - } - - // Add resource to the cache - if (!dev().resourceCache().addGpuMemory(&desc_, memRef_)) { - palFree(); - } + if (wait) { + if (memRef_->gpu_ == nullptr) { + Device::ScopedLockVgpus lock(dev()); + // Release all memory objects on all virtual GPUs + for (uint idx = 1; idx < dev().vgpus().size(); ++idx) { + dev().vgpus()[idx]->waitForEvent(&memRef_->events_[idx]); } - } else { - renames_[curRename_]->cpuAddress_ = 0; - for (size_t i = 0; i < renames_.size(); ++i) { - memRef_ = renames_[i]; - // Destroy GSL resource - if (iMem() != 0) { - // Release all virtual memory objects on all virtual GPUs - for (uint idx = 0; idx < dev().vgpus().size(); ++idx) { - // Ignore the transfer queue, - // since it releases resources after every operation - if (dev().vgpus()[idx] != dev().xferQueue()) { - dev().vgpus()[idx]->releaseMemory(iMem()); - } - } - palFree(); - } + } + else { + memRef_->gpu_->waitForEvent(&memRef_->events_[memRef_->gpu_->index()]); + } + } + + if (renames_.size() == 0) { + // Destroy GSL resource + if (iMem() != 0) { + //! @note: This is a workaround for bad applications that + //! don't unmap memory + if (mapCount_ != 0) { + unmap(nullptr); + } + + // Add resource to the cache if it's not assigned to a specific queue + if ((memRef_->gpu_ != nullptr) || !dev().resourceCache().addGpuMemory(&desc_, memRef_)) { + palFree(); } } } else { - if (renames_.size() == 0) { - // Destroy GSL resource + renames_[curRename_]->cpuAddress_ = 0; + for (size_t i = 0; i < renames_.size(); ++i) { + memRef_ = renames_[i]; + // Destroy PAL resource if (iMem() != 0) { - // Release virtual memory object on the specified virtual GPU - gpu_->releaseMemory(iMem(), wait); palFree(); } - } else - for (size_t i = 0; i < renames_.size(); ++i) { - memRef_ = renames_[i]; - // Destroy GSL resource - if (iMem() != 0) { - // Release virtual memory object on the specified virtual GPUs - gpu_->releaseMemory(iMem()); - palFree(); - } - } + } } // Free SRD for images @@ -1150,7 +1139,7 @@ void Resource::writeRawData(VirtualGPU& gpu, size_t offset, size_t size, const v // size needs to be DWORD aligned assert((size & 3) == 0); gpu.eventBegin(MainEngine); - gpu.queue(MainEngine).addCmdMemRef(iMem()); + gpu.queue(MainEngine).addCmdMemRef(memRef()); gpu.iCmd()->CmdUpdateMemory(*iMem(), offset, size, reinterpret_cast(data)); gpu.eventEnd(MainEngine, event); @@ -1244,8 +1233,8 @@ bool Resource::partialMemCopyTo(VirtualGPU& gpu, const amd::Coord3D& srcOrigin, Pal::ImageLayout imgLayout = {}; gpu.eventBegin(gpu.engineID_); - gpu.queue(gpu.engineID_).addCmdMemRef(iMem()); - gpu.queue(gpu.engineID_).addCmdMemRef(dstResource.iMem()); + gpu.queue(gpu.engineID_).addCmdMemRef(memRef()); + gpu.queue(gpu.engineID_).addCmdMemRef(dstResource.memRef()); if (desc().buffer_ && !dstResource.desc().buffer_) { Pal::SubresId ImgSubresId = {Pal::ImageAspect::Color, dstResource.desc().baseLevel_, 0}; Pal::MemoryImageCopyRegion copyRegion = {}; @@ -1340,7 +1329,7 @@ bool Resource::partialMemCopyTo(VirtualGPU& gpu, const amd::Coord3D& srcOrigin, } void Resource::setBusy(VirtualGPU& gpu, GpuEvent gpuEvent) const { - gpu.assignGpuEvent(iMem(), gpuEvent); + addGpuEvent(gpu, gpuEvent); // If current resource is a view, then update the parent event as well if (viewOwner_ != nullptr) { @@ -1349,7 +1338,7 @@ void Resource::setBusy(VirtualGPU& gpu, GpuEvent gpuEvent) const { } void Resource::wait(VirtualGPU& gpu, bool waitOnBusyEngine) const { - GpuEvent* gpuEvent = gpu.getGpuEvent(iMem()); + GpuEvent* gpuEvent = getGpuEvent(gpu); // Check if we have to wait unconditionally if (!waitOnBusyEngine || @@ -1560,10 +1549,22 @@ bool Resource::glRelease() { } return retVal; } -void Resource::palFree() const { - amd::ScopedLock lk(dev().lockPAL()); +void Resource::addGpuEvent(const VirtualGPU& gpu, GpuEvent event) const { + uint idx = gpu.index(); + assert(idx < memRef_->events_.size()); + memRef_->events_[idx] = event; +} + +GpuEvent* Resource::getGpuEvent(const VirtualGPU& gpu) const { + uint idx = gpu.index(); + assert((idx < memRef_->events_.size()) && "Undeclared queue access!"); + return &memRef_->events_[idx]; +} + +void Resource::palFree() const { if (desc().type_ == OGLInterop) { + amd::ScopedLock lk(dev().lockPAL()); dev().resGLFree(glPlatformContext_, glInteropMbRes_, glType_); } memRef_->release(); @@ -1857,7 +1858,7 @@ bool Resource::getActiveRename(VirtualGPU& gpu, GpuMemoryReference** rename) { } bool Resource::rename(VirtualGPU& gpu, bool force) { - GpuEvent* gpuEvent = gpu.getGpuEvent(iMem()); + GpuEvent* gpuEvent = getGpuEvent(gpu); if (!gpuEvent->isValid() && !force) { return true; } diff --git a/projects/clr/rocclr/runtime/device/pal/palresource.hpp b/projects/clr/rocclr/runtime/device/pal/palresource.hpp index dc8d55d915..36156683af 100644 --- a/projects/clr/rocclr/runtime/device/pal/palresource.hpp +++ b/projects/clr/rocclr/runtime/device/pal/palresource.hpp @@ -34,13 +34,24 @@ class GpuMemoryReference : public amd::ReferenceCountedObject { Pal::ImageCreateInfo* imgCreateInfo, Pal::IImage** image); //! Default constructor - GpuMemoryReference(); + GpuMemoryReference(const Device& dev); + + //! Resizes the events array to account the new queue + void resizeGpuEvents(uint index) { events_.resize(index + 1); } + + //! Erase an entry in the array for provided queue index + void eraseGpuEvents(uint index) { events_.erase(events_.begin() + index); } //! Get PAL memory object Pal::IGpuMemory* iMem() const { return gpuMem_; } - Pal::IGpuMemory* gpuMem_; //!< PAL GPU memory object - void* cpuAddress_; //!< CPU address of this memory + Pal::IGpuMemory* gpuMem_; //!< PAL GPU memory object + void* cpuAddress_; //!< CPU address of this memory + const Device& device_; //!< GPU device + //! @note: This field is necessary for the thread safe release only + VirtualGPU* gpu_; //!< Resource will be used only on this queue + std::vector events_; //!< GPU events associated with the resource + std::atomic resident_; //!< Atomic counter for residency protected: //! Default destructor @@ -105,8 +116,6 @@ class Resource : public amd::HeapObject { uint mipLevel_; //!< Texture mip level uint layer_; //!< Texture layer void* glPlatformContext_; - void* glDeviceContext_; - uint flags_; }; #ifdef _WIN32 @@ -251,6 +260,9 @@ class Resource : public amd::HeapObject { //! Returns the PAL memory object Pal::IGpuMemory* iMem() const { return memRef_->iMem(); } + //! Returns a pointer to the memory reference + GpuMemoryReference* memRef() const {return memRef_; } + //! Returns global memory offset uint64_t vmAddress() const { return iMem()->Desc().gpuVirtAddr + offset_; } @@ -341,10 +353,17 @@ class Resource : public amd::HeapObject { //! Returns CPU HW SRD for the resource (used for images only) uint64_t hwSrd() const { return hwSrd_; } + //! Returns the number of components in the image format uint numComponents() const { return Pal::Formats::NumComponents(image_->GetImageCreateInfo().swizzledFormat.format); } + //! Adds GPU event, associated with this resource + void addGpuEvent(const VirtualGPU& gpu, GpuEvent event) const; + + //! Returns GPU event associated with this resource and specified queue + GpuEvent* getGpuEvent(const VirtualGPU& gpu) const; + protected: uint elementSize_; //!< Size of a single element in bytes @@ -406,7 +425,7 @@ class Resource : public amd::HeapObject { amd::Atomic mapCount_; //!< Total number of maps void* address_; //!< Physical address of this resource size_t offset_; //!< Resource offset - size_t curRename_; //!< Current active rename in the list + uint32_t curRename_; //!< Current active rename in the list RenameList renames_; //!< Rename resource list GpuMemoryReference* memRef_; //!< PAL resource reference const Resource* viewOwner_; //!< GPU resource, which owns this view @@ -414,15 +433,12 @@ class Resource : public amd::HeapObject { void* glInteropMbRes_; //!< Mb Res handle uint32_t glType_; //!< GL interop type void* glPlatformContext_; - void* glDeviceContext_; // Optimization for multilayer map/unmap uint startLayer_; //!< Start layer for map/unmapLayer uint numLayers_; //!< Number of layers for map/unmapLayer uint mapFlags_; //!< Map flags for map/umapLayer - //! @note: This field is necessary for the thread safe release only - VirtualGPU* gpu_; //!< Resource will be used only on this queue Pal::IImage* image_; //!< PAL image object uint32_t* hwState_; //!< HW state for image object diff --git a/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp b/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp index cd03ade47d..aea220fe4e 100644 --- a/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/pal/palvirtual.cpp @@ -34,7 +34,8 @@ namespace pal { VirtualGPU::Queue* VirtualGPU::Queue::Create(Pal::IDevice* palDev, Pal::QueueType queueType, uint engineIdx, Pal::ICmdAllocator* cmdAllocator, - uint rtCU, amd::CommandQueue::Priority priority) { + uint rtCU, amd::CommandQueue::Priority priority, + uint64_t residency_limit) { Pal::Result result; Pal::CmdBufferCreateInfo cmdCreateInfo = {}; Pal::QueueCreateInfo qCreateInfo = {}; @@ -80,7 +81,7 @@ VirtualGPU::Queue* VirtualGPU::Queue::Create(Pal::IDevice* palDev, Pal::QueueTyp } size_t allocSize = qSize + MaxCmdBuffers * (cmdSize + fSize); - VirtualGPU::Queue* queue = new (allocSize) VirtualGPU::Queue(palDev); + VirtualGPU::Queue* queue = new (allocSize) VirtualGPU::Queue(palDev, residency_limit); if (queue != nullptr) { address addrQ = reinterpret_cast
(&queue[1]); // Create PAL queue object @@ -119,10 +120,10 @@ VirtualGPU::Queue* VirtualGPU::Queue::Create(Pal::IDevice* palDev, Pal::QueueTyp } VirtualGPU::Queue::~Queue() { - std::vector memRef; // Remove all memory references + std::vector memRef; for (auto it : memReferences_) { - memRef.push_back(it.first); + memRef.push_back(it.first->iMem()); } if (memRef.size() != 0) { iDev_->RemoveGpuMemoryReferences(memRef.size(), &memRef[0], NULL); @@ -143,18 +144,35 @@ VirtualGPU::Queue::~Queue() { } } -void VirtualGPU::Queue::addCmdMemRef(Pal::IGpuMemory* iMem) { - auto it = memReferences_.find(iMem); +void VirtualGPU::Queue::addCmdMemRef(GpuMemoryReference* mem) { + Pal::IGpuMemory* iMem = mem->iMem(); + auto it = memReferences_.find(mem); if (it != memReferences_.end()) { - it->second = (it->second & FirstMemoryReference) | cmdBufIdSlot_; + it->second = cmdBufIdSlot_; } else { - memReferences_[iMem] = FirstMemoryReference | cmdBufIdSlot_; + // Update runtime tracking with TS + memReferences_[mem] = cmdBufIdSlot_; + // Update PAL list with the new entry + Pal::GpuMemoryRef memRef = {}; + memRef.pGpuMemory = iMem; + palMemRefs_.push_back(memRef); + // Check SDI memory object + if (iMem->Desc().flags.isExternPhys && + (sdiReferences_.find(iMem) == sdiReferences_.end())) { + sdiReferences_.insert(iMem); + palSdiRefs_.push_back(iMem); + } + residency_size_ += iMem->Desc().size; + mem->resident_++; } } -void VirtualGPU::Queue::removeCmdMemRef(Pal::IGpuMemory* iMem) { - if (0 != memReferences_.erase(iMem)) { +void VirtualGPU::Queue::removeCmdMemRef(GpuMemoryReference* mem) { + Pal::IGpuMemory* iMem = mem->iMem(); + if (0 != memReferences_.erase(mem)) { iDev_->RemoveGpuMemoryReferences(1, &iMem, iQueue_); + residency_size_ -= iMem->Desc().size; + mem->resident_--; } } @@ -198,26 +216,16 @@ uint VirtualGPU::Queue::submit(bool forceFlush) { } bool VirtualGPU::Queue::flush() { - palMemRefs_.resize(0); // Stop commands building if (Pal::Result::Success != iCmdBuffs_[cmdBufIdSlot_]->End()) { LogError("PAL failed to finalize a command buffer!"); return false; } - // Add memory references - for (auto it = memReferences_.begin(); it != memReferences_.end(); ++it) { - if (it->second & FirstMemoryReference) { - it->second &= ~FirstMemoryReference; - Pal::GpuMemoryRef memRef = {}; - memRef.pGpuMemory = it->first; - palMemRefs_.push_back(memRef); - - if (it->first->Desc().flags.isExternPhys - && (sdiReferences_.find(it->first) == sdiReferences_.end())) { - sdiReferences_.insert(it->first); - palSdiRefs_.push_back(it->first); - } + // Validate resources + for (auto it : memReferences_) { + if (it.second == cmdBufIdSlot_) { + assert(it.first->resident_ > 0 && "Unresident resource!"); } } @@ -228,6 +236,7 @@ bool VirtualGPU::Queue::flush() { LogError("PAL failed to make resident resources!"); return false; } + palMemRefs_.resize(0); } // Reset the fence. PAL will reset OS event @@ -295,21 +304,24 @@ bool VirtualGPU::Queue::flush() { // Clear dopp references palDoppRefs_.resize(0); - - palMems_.resize(0); palSdiRefs_.resize(0); // Remove old memory references - for (auto it = memReferences_.begin(); it != memReferences_.end();) { - if (it->second == cmdBufIdSlot_) { - palMems_.push_back(it->first); - it = memReferences_.erase(it); - } else { - ++it; + if ((memReferences_.size() > 1024) || (residency_size_ > residency_limit_)) { + for (auto it = memReferences_.begin(); it != memReferences_.end();) { + if (it->second == cmdBufIdSlot_) { + palMems_.push_back(it->first->iMem()); + residency_size_ -= it->first->iMem()->Desc().size; + it->first->resident_--; + it = memReferences_.erase(it); + } else { + ++it; + } } } if (palMems_.size() != 0) { iDev_->RemoveGpuMemoryReferences(palMems_.size(), &palMems_[0], iQueue_); + palMems_.resize(0); } return true; @@ -370,8 +382,8 @@ void VirtualGPU::Queue::DumpMemoryReferences() const { dump << " " << idx << "\t["; dump.setf(std::ios::hex, std::ios::basefield); dump.setf(std::ios::showbase); - dump << (it.first)->Desc().gpuVirtAddr << ", " - << (it.first)->Desc().gpuVirtAddr + (it.first)->Desc().size; + dump << (it.first)->iMem()->Desc().gpuVirtAddr << ", " + << (it.first)->iMem()->Desc().gpuVirtAddr + (it.first)->iMem()->Desc().size; dump.setf(std::ios::dec); dump << "] CbId:" << it.second << "\n"; idx++; @@ -556,7 +568,7 @@ void VirtualGPU::addPinnedMem(amd::Memory* mem) { } // Start operation, since we should release mem object - flushDMA(getGpuEvent(dev().getGpuMemory(mem)->iMem())->engineId_); + flushDMA(dev().getGpuMemory(mem)->getGpuEvent(*this)->engineId_); // Delay destruction pinnedMems_.push_back(mem); @@ -721,6 +733,7 @@ VirtualGPU::VirtualGPU(Device& device) index_ = gpuDevice_.numOfVgpus_++; gpuDevice_.vgpus_.resize(gpuDevice_.numOfVgpus()); gpuDevice_.vgpus_[index()] = this; + queues_[MainEngine] = nullptr; queues_[SdmaEngine] = nullptr; } @@ -734,6 +747,8 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs, return false; } + dev().resizeResoureList(index()); + // Virtual GPU will have profiling enabled state_.profiling_ = profiling; @@ -742,8 +757,9 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs, // \todo forces PAL to reuse CBs, but requires postamble createInfo.flags.autoMemoryReuse = false; createInfo.allocInfo[Pal::CommandDataAlloc].allocHeap = Pal::GpuHeapGartCacheable; - createInfo.allocInfo[Pal::CommandDataAlloc].allocSize = 128 * Ki; - createInfo.allocInfo[Pal::CommandDataAlloc].suballocSize = 128 * Ki; + createInfo.allocInfo[Pal::CommandDataAlloc].allocSize = + createInfo.allocInfo[Pal::CommandDataAlloc].suballocSize = + VirtualGPU::Queue::MaxCommands * (256 + ((profiling) ? 64 : 0)); createInfo.allocInfo[Pal::EmbeddedDataAlloc].allocHeap = Pal::GpuHeapGartCacheable; createInfo.allocInfo[Pal::EmbeddedDataAlloc].allocSize = 64 * Ki; @@ -761,6 +777,8 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs, const uint firstQueue = (dev().numComputeEngines() > 2) ? 1 : 0; uint idx = index() % (dev().numComputeEngines() - firstQueue); + uint64_t residency_limit = dev().properties().gpuMemoryProperties.flags.supportPerSubmitMemRefs ? 0 : + (dev().properties().gpuMemoryProperties.maxLocalMemSize >> 2); if (dev().numComputeEngines()) { //! @todo There is a hang with a mix of user and non user queues. @@ -771,7 +789,8 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs, hwRing_ = (dev().settings().useSingleScratch_) ? 0 : idx; queues_[MainEngine] = Queue::Create(dev().iDev(), Pal::QueueTypeCompute, idx + firstQueue, - cmdAllocator_, rtCUs, priority); + cmdAllocator_, rtCUs, priority, + residency_limit); if (nullptr == queues_[MainEngine]) { return false; } @@ -788,13 +807,15 @@ bool VirtualGPU::create(bool profiling, uint deviceQueueSize, uint rtCUs, queues_[SdmaEngine] = Queue::Create(dev().iDev(), Pal::QueueTypeDma, sdma, cmdAllocator_, - amd::CommandQueue::RealTimeDisabled, amd::CommandQueue::Priority::Normal); + amd::CommandQueue::RealTimeDisabled, amd::CommandQueue::Priority::Normal, + residency_limit); if (nullptr == queues_[SdmaEngine]) { return false; } } else { queues_[SdmaEngine] = Queue::Create(dev().iDev(), Pal::QueueTypeCompute, - idx, cmdAllocator_, rtCUs, amd::CommandQueue::Priority::Normal); + idx, cmdAllocator_, rtCUs, amd::CommandQueue::Priority::Normal, + residency_limit); if (nullptr == queues_[SdmaEngine]) { return false; } @@ -905,10 +926,6 @@ VirtualGPU::~VirtualGPU() { amd::ScopedLock k(dev().lockAsyncOps()); amd::ScopedLock lock(dev().vgpusAccess()); - // Destroy all memories - static const bool SkipScratch = false; - releaseMemObjects(SkipScratch); - while (!freeCbQueue_.empty()) { auto cb = freeCbQueue_.front(); delete cb; @@ -921,9 +938,6 @@ VirtualGPU::~VirtualGPU() { // Destroy printfHSA object delete printfDbgHSA_; - // Destroy BlitManager object - delete blitMgr_; - // Destroy TimeStamp cache delete tsCache_; @@ -932,46 +946,73 @@ VirtualGPU::~VirtualGPU() { delete constBufs_[i]; } - // Destroy queues - if (nullptr != queues_[MainEngine]) { - // Make sure the queues are idle - // It's unclear why PAL could still have a busy queue - queues_[MainEngine]->iQueue_->WaitIdle(); - delete queues_[MainEngine]; + //! @todo Temporarily keep the buffer mapped for debug purpose + if (nullptr != schedParams_) { + schedParams_->unmap(this); } - if (nullptr != queues_[SdmaEngine]) { - queues_[SdmaEngine]->iQueue_->WaitIdle(); - delete queues_[SdmaEngine]; - } - - if (nullptr != cmdAllocator_) { - cmdAllocator_->Destroy(); - delete[] reinterpret_cast(cmdAllocator_); - } - - gpuDevice_.numOfVgpus_--; - gpuDevice_.vgpus_.erase(gpuDevice_.vgpus_.begin() + index()); - for (uint idx = index(); idx < dev().vgpus().size(); ++idx) { - dev().vgpus()[idx]->index_--; - } + delete vqHeader_; + delete virtualQueue_; + delete schedParams_; + delete hsaQueueMem_; // Release scratch buffer memory to reduce memory pressure //!@note OCLtst uses single device with multiple tests //! Release memory only if it's the last command queue. //! The first queue is reserved for the transfers on device - if (gpuDevice_.numOfVgpus_ <= 1) { + if (static_cast(gpuDevice_.numOfVgpus_ - 1) <= 1) { gpuDevice_.destroyScratchBuffers(); } - //! @todo Temporarily keep the buffer mapped for debug purpose - if (nullptr != schedParams_) { - schedParams_->unmap(this); + // Destroy BlitManager object + delete blitMgr_; + + { + // Destroy queues + if (nullptr != queues_[MainEngine]) { + // Make sure the queues are idle + // It's unclear why PAL could still have a busy queue + queues_[MainEngine]->iQueue_->WaitIdle(); + delete queues_[MainEngine]; + } + + if (nullptr != queues_[SdmaEngine]) { + queues_[SdmaEngine]->iQueue_->WaitIdle(); + delete queues_[SdmaEngine]; + } + + if (nullptr != cmdAllocator_) { + cmdAllocator_->Destroy(); + delete[] reinterpret_cast(cmdAllocator_); + } + } + + { + // Find all available virtual GPUs and lock them + // from the execution of commands, since the queue index and resource list + // Will be adjusted + for (auto it : dev().vgpus()) { + if (it != this) { + it->execution().lock(); + } + } + + // Not safe to add a resource if create/destroy queue is in process, since + // the size of the TS array can change + amd::ScopedLock r(dev().lockResources()); + gpuDevice_.numOfVgpus_--; + gpuDevice_.vgpus_.erase(gpuDevice_.vgpus_.begin() + index()); + for (uint idx = index(); idx < dev().vgpus().size(); ++idx) { + dev().vgpus()[idx]->index_--; + } + dev().eraseResoureList(index()); + + // Find all available virtual GPUs and unlock them + // for the execution of commands + for (auto it : dev().vgpus()) { + it->execution().unlock(); + } } - delete vqHeader_; - delete virtualQueue_; - delete schedParams_; - delete hsaQueueMem_; } void VirtualGPU::submitReadMemory(amd::ReadMemoryCommand& vcmd) { @@ -1859,11 +1900,8 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const VirtualGPU* gpuDefQueue = nullptr; amd::HwDebugManager* dbgManager = dev().hwDebugMgr(); - AddKernel(kernel); - // Get the HSA kernel object const HSAILKernel& hsaKernel = static_cast(*(kernel.getDeviceKernel(dev()))); - std::vector dispMemList; //!< Memory list of all mem objects used in the disaptch bool printfEnabled = (hsaKernel.printfInfo().size() > 0) ? true : false; if (!printfDbgHSA().init(*this, printfEnabled)) { @@ -1872,11 +1910,13 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const } // Check memory dependency and SVM objects - if (!processMemObjectsHSA(kernel, parameters, nativeMem, &dispMemList)) { + if (!processMemObjectsHSA(kernel, parameters, nativeMem)) { LogError("Wrong memory objects!"); return false; } + AddKernel(kernel); + if (hsaKernel.dynamicParallelism()) { if (nullptr == defQueue) { LogError("Default device queue wasn't allocated"); @@ -1895,11 +1935,11 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const } vmDefQueue = gpuDefQueue->virtualQueue_->vmAddress(); - // Add memory handles before the actual dispatch - dispMemList.push_back(gpuDefQueue->virtualQueue_); - dispMemList.push_back(gpuDefQueue->schedParams_); - dispMemList.push_back(hsaKernel.prog().kernelTable()); gpuDefQueue->writeVQueueHeader(*this, hsaKernel.prog().kernelTable()->vmAddress()); + // Add memory handles before the actual dispatch + addVmMemory(gpuDefQueue->virtualQueue_); + addVmMemory(gpuDefQueue->schedParams_); + addVmMemory(hsaKernel.prog().kernelTable()); } // setup the storage for the memory pointers of the kernel parameters @@ -1940,8 +1980,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const } } } - for (int j = 0; j < iteration; j++) { + GpuEvent gpuEvent(queues_[MainEngine]->cmdBufId()); + uint32_t id = gpuEvent.id; // Reset global size for dimension dim if split is needed if (dim != -1) { newOffset[dim] = sizes.offset()[dim] + globalStep * j; @@ -1957,7 +1998,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const // Program the kernel arguments for the GPU execution hsa_kernel_dispatch_packet_t* aqlPkt = hsaKernel.loadArguments( - *this, kernel, tmpSizes, parameters, nativeMem, vmDefQueue, &vmParentWrap, dispMemList); + *this, kernel, tmpSizes, parameters, nativeMem, vmDefQueue, &vmParentWrap); if (nullptr == aqlPkt) { LogError("Couldn't load kernel arguments"); return false; @@ -1967,16 +2008,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const // Check if the device allocated more registers than the old setup if (hsaKernel.workGroupInfo()->scratchRegs_ > 0) { scratch = dev().scratch(hwRing()); - dispMemList.push_back(scratch->memObj_); - } - - // Add GSL handle to the memory list for VidMM - for (uint i = 0; i < dispMemList.size(); ++i) { - addVmMemory(dispMemList[i]); - if (dispMemList[i]->desc().isDoppTexture_) { - addDoppRef(dispMemList[i], kernel.parameters().getExecNewVcop(), - kernel.parameters().getExecPfpaVcop()); - } + addVmMemory(scratch->memObj_); } // HW Debug for the kernel? @@ -1988,7 +2020,6 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const pKernelInfo = &kernelInfo; } - GpuEvent gpuEvent; // Set up the dispatch information Pal::DispatchAqlParams dispatchParam = {}; dispatchParam.pAqlPacket = aqlPkt; @@ -2005,7 +2036,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const eventBegin(MainEngine); iCmd()->CmdDispatchAql(dispatchParam); eventEnd(MainEngine, gpuEvent); - + if (id != gpuEvent.id) { + LogError("something is wrong. ID mismatch!\n"); + } if (dbgManager && (nullptr != dbgManager->postDispatchCallBackFunc())) { dbgManager->executePostDispatchCallBack(); } @@ -2166,7 +2199,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const param->scratch = scratchBuf->vmAddress(); param->numMaxWaves = 32 * dev().info().maxComputeUnits_; param->scratchOffset = dev().scratch(gpuDefQueue->hwRing())->offset_; - dispMemList.push_back(scratchBuf); + addVmMemory(scratchBuf); } else { param->numMaxWaves = 0; param->scratchSize = 0; @@ -2176,12 +2209,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const // Add all kernels in the program to the mem list. //! \note Runtime doesn't know which one will be called - hsaKernel.prog().fillResListWithKernels(dispMemList); - - // Add GPU memory handle to the memory list for VidMM - for (uint i = 0; i < dispMemList.size(); ++i) { - gpuDefQueue->addVmMemory(dispMemList[i]); - } + hsaKernel.prog().fillResListWithKernels(*this); Pal::gpusize signalAddr = gpuDefQueue->schedParams_->vmAddress() + gpuDefQueue->schedParamIdx_ * sizeof(SchedulerParam); @@ -2194,11 +2222,6 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const constexpr bool ForceSubmitFirst = true; gpuDefQueue->eventEnd(MainEngine, gpuEvent, ForceSubmitFirst); - // Set GPU event for the used resources - for (uint i = 0; i < dispMemList.size(); ++i) { - dispMemList[i]->setBusy(*gpuDefQueue, gpuEvent); - } - if (dev().settings().useDeviceQueue_) { // Add the termination handshake to the host queue eventBegin(MainEngine); @@ -2214,12 +2237,9 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, const gpuDefQueue->schedParams_->wait(*gpuDefQueue); } } - - // Set GPU event for the used resources - for (uint i = 0; i < dispMemList.size(); ++i) { - dispMemList[i]->setBusy(*this, gpuEvent); + if (id != gpuEvent.id) { + LogError("Something is wrong. ID mismatch!\n"); } - // Update the global GPU event setGpuEvent(gpuEvent, needFlush); @@ -2269,27 +2289,10 @@ void VirtualGPU::submitMarker(amd::Marker& vcmd) { } } -GpuEvent* VirtualGPU::getGpuEvent(Pal::IGpuMemory* iMem) { return &gpuEvents_[iMem]; } - -void VirtualGPU::assignGpuEvent(Pal::IGpuMemory* iMem, GpuEvent gpuEvent) { - auto it = gpuEvents_.find(iMem); - - if (it != gpuEvents_.end()) { - it->second = gpuEvent; - } else { - gpuEvents_[iMem] = gpuEvent; - } -} - -void VirtualGPU::releaseMemory(Pal::IGpuMemory* iMem, bool wait) { - auto it = gpuEvents_.find(iMem); - //! @note if there is no wait, then it's a view release - if (wait && (it != gpuEvents_.end())) { - waitForEvent(&it->second); - queues_[MainEngine]->removeCmdMemRef(iMem); - queues_[SdmaEngine]->removeCmdMemRef(iMem); - gpuEvents_.erase(it); - } +void VirtualGPU::releaseMemory(GpuMemoryReference* mem, GpuEvent* event) { + waitForEvent(event); + queues_[MainEngine]->removeCmdMemRef(mem); + queues_[SdmaEngine]->removeCmdMemRef(mem); } void VirtualGPU::submitPerfCounter(amd::PerfCounterCommand& vcmd) { @@ -2554,16 +2557,14 @@ void VirtualGPU::submitSignal(amd::SignalCommand& vcmd) { uint32_t value = vcmd.markerValue(); addVmMemory(pGpuMemory); - if (vcmd.type() == CL_COMMAND_WAIT_SIGNAL_AMD) { iCmd()->CmdWaitBusAddressableMemoryMarker(*(pGpuMemory->iMem()), value, 0xFFFFFFFF, Pal::CompareFunc::GreaterEqual); } else if (vcmd.type() == CL_COMMAND_WRITE_SIGNAL_AMD) { iCmd()->CmdUpdateBusAddressableMemoryMarker(*(pGpuMemory->iMem()), value); } - eventEnd(MainEngine, gpuEvent); - pGpuMemory->setBusy(*this, gpuEvent); + // Update the global GPU event setGpuEvent(gpuEvent); @@ -2717,17 +2718,6 @@ void VirtualGPU::flush(amd::Command* list, bool wait) { void VirtualGPU::enableSyncedBlit() const { return blitMgr_->enableSynchronization(); } -void VirtualGPU::releaseMemObjects(bool scratch) { - for (GpuEvents::const_iterator it = gpuEvents_.begin(); it != gpuEvents_.end(); ++it) { - GpuEvent event = it->second; - waitForEvent(&event); - queues_[MainEngine]->removeCmdMemRef(const_cast(it->first)); - queues_[SdmaEngine]->removeCmdMemRef(const_cast(it->first)); - } - - gpuEvents_.clear(); -} - void VirtualGPU::setGpuEvent(GpuEvent gpuEvent, bool flush) { cal_.events_[engineID_] = gpuEvent; @@ -2784,7 +2774,6 @@ bool VirtualGPU::waitAllEngines(CommandBatch* cb) { void VirtualGPU::waitEventLock(CommandBatch* cb) { // Make sure VirtualGPU has an exclusive access to the resources amd::ScopedLock lock(execution()); - bool earlyDone = waitAllEngines(cb); // Free resource cache if we have too many entries @@ -2954,8 +2943,11 @@ bool VirtualGPU::profilingCollectResults(CommandBatch* cb, const amd::Event* wai } void VirtualGPU::addVmMemory(const Memory* memory) { - queues_[MainEngine]->addCmdMemRef(memory->iMem()); -} + GpuEvent event(queues_[MainEngine]->cmdBufId()); + queues_[MainEngine]->addCmdMemRef(memory->memRef()); + memory->setBusy(*this, event); +} + void VirtualGPU::AddKernel(const amd::Kernel& kernel) const { queues_[MainEngine]->last_kernel_ = &kernel; } @@ -2976,13 +2968,13 @@ void VirtualGPU::profileEvent(EngineType engine, bool type) const { } bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address params, - bool nativeMem, std::vector* memList) { + bool nativeMem) { static const bool NoAlias = true; const HSAILKernel& hsaKernel = static_cast(*(kernel.getDeviceKernel(dev(), NoAlias))); const amd::KernelSignature& signature = kernel.signature(); const amd::KernelParameters& kernelParams = kernel.parameters(); - + std::vector memList; // Mark the tracker with a new kernel, // so we can avoid checks of the aliased objects memoryDependency().newKernel(); @@ -3040,13 +3032,16 @@ bool VirtualGPU::processMemObjectsHSA(const amd::Kernel& kernel, const_address p memory->signalWrite(&dev()); } - memList->push_back(gpuMemory); + memList.push_back(gpuMemory); } else { return false; } } } + for (auto it : memList) { + addVmMemory(it); + } // Check all parameters for the current kernel for (size_t i = 0; i < signature.numParameters(); ++i) { const amd::KernelParameterDescriptor& desc = signature.at(i); @@ -3264,7 +3259,6 @@ void VirtualGPU::assignDebugTrapHandler(const DebugToolInfo& dbgSetting, rtTmaPtr[1] = tmaAddress; rtTrapBufferMem->unmap(nullptr); - // Add GPU mem handles to the memory list for VidMM addVmMemory(trapHandlerMem); addVmMemory(trapBufferMem); @@ -3312,7 +3306,7 @@ void VirtualGPU::submitTransferBufferFromFile(amd::TransferBufferFileCommand& cm staging->cpuUnmap(*this); bool result = blitMgr().copyBuffer(*staging, *mem, 0, dstOffset, dstSize, false); - flushDMA(getGpuEvent(staging->iMem())->engineId_); + flushDMA(staging->getGpuEvent(*this)->engineId_); fileOffset += dstSize; dstOffset += dstSize; copySize -= dstSize; diff --git a/projects/clr/rocclr/runtime/device/pal/palvirtual.hpp b/projects/clr/rocclr/runtime/device/pal/palvirtual.hpp index 4eabc1a902..19f93202da 100644 --- a/projects/clr/rocclr/runtime/device/pal/palvirtual.hpp +++ b/projects/clr/rocclr/runtime/device/pal/palvirtual.hpp @@ -56,10 +56,11 @@ class VirtualGPU : public device::VirtualDevice { uint engineIdx, //!< Select particular engine index Pal::ICmdAllocator* cmdAlloc, //!< PAL CMD buffer allocator uint rtCU, //!< The number of reserved CUs - amd::CommandQueue::Priority priority //!< Queue priority + amd::CommandQueue::Priority priority, //!< Queue priority + uint64_t residency_limit //!< Enables residency limit ); - Queue(Pal::IDevice* palDev) + Queue(Pal::IDevice* palDev, uint64_t residency_limit) : iQueue_(nullptr), last_kernel_(nullptr), iDev_(palDev), @@ -67,7 +68,10 @@ class VirtualGPU : public device::VirtualDevice { cmdBufIdCurrent_(StartCmdBufIdx), cmbBufIdRetired_(0), cmdCnt_(0), - vlAlloc_(64 * Ki) { + vlAlloc_(64 * Ki), + residency_size_(0), + residency_limit_(residency_limit) + { for (uint i = 0; i < MaxCmdBuffers; ++i) { iCmdBuffs_[i] = nullptr; iCmdFences_[i] = nullptr; @@ -77,8 +81,8 @@ class VirtualGPU : public device::VirtualDevice { ~Queue(); - void addCmdMemRef(Pal::IGpuMemory* iMem); - void removeCmdMemRef(Pal::IGpuMemory* iMem); + void addCmdMemRef(GpuMemoryReference* mem); + void removeCmdMemRef(GpuMemoryReference* mem); void addCmdDoppRef(Pal::IGpuMemory* iMem, bool lastDoppCmd, bool pfpaDoppCmd); @@ -133,6 +137,8 @@ class VirtualGPU : public device::VirtualDevice { Pal::ICmdBuffer* iCmd() const { return iCmdBuffs_[cmdBufIdSlot_]; } + uint cmdBufId() const { return cmdBufIdCurrent_; } + Pal::IQueue* iQueue_; //!< PAL queue object Pal::ICmdBuffer* iCmdBuffs_[MaxCmdBuffers]; //!< PAL command buffers Pal::IFence* iCmdFences_[MaxCmdBuffers]; //!< PAL fences, associated with CMD @@ -145,13 +151,15 @@ class VirtualGPU : public device::VirtualDevice { uint cmdBufIdCurrent_; //!< Current global command buffer ID uint cmbBufIdRetired_; //!< The last retired command buffer ID uint cmdCnt_; //!< Counter of commands - std::map memReferences_; + std::map memReferences_; Util::VirtualLinearAllocator vlAlloc_; std::vector palMemRefs_; std::vector palMems_; std::vector palDoppRefs_; std::set sdiReferences_; std::vector palSdiRefs_; + uint64_t residency_size_; //!< Resource residency size + uint64_t residency_limit_; //!< Enables residency limit }; struct CommandBatch : public amd::HeapObject { @@ -303,7 +311,7 @@ class VirtualGPU : public device::VirtualDevice { virtual void submitSvmUnmapMemory(amd::SvmUnmapMemoryCommand& cmd); virtual void submitTransferBufferFromFile(amd::TransferBufferFileCommand& cmd); - void releaseMemory(Pal::IGpuMemory* iMem, bool wait = true); + void releaseMemory(GpuMemoryReference* mem, GpuEvent* event); void flush(amd::Command* list = nullptr, bool wait = false); bool terminate() { return true; } @@ -314,14 +322,6 @@ class VirtualGPU : public device::VirtualDevice { //! Returns CAL descriptor of the virtual device const CalVirtualDesc* cal() const { return &cal_; } - //! Returns a GPU event, associated with GPU memory - GpuEvent* getGpuEvent(Pal::IGpuMemory* iMem //!< PAL mem object - ); - - //! Assigns a GPU event, associated with GPU memory - void assignGpuEvent(Pal::IGpuMemory* iMem, //!< PAL mem object - GpuEvent gpuEvent); - //! Set the last known GPU event void setGpuEvent(GpuEvent gpuEvent, //!< GPU event for tracking bool flush = false //!< TRUE if flush is required @@ -362,12 +362,12 @@ class VirtualGPU : public device::VirtualDevice { ); //! Adds a memory handle into the GSL memory array for Virtual Heap - void addVmMemory(const Memory* memory //!< GPU memory object - ); + inline void addVmMemory(const Memory* memory //!< GPU memory object + ); //! Adds the last submitted kernel to the queue for tracking a possible hang - void AddKernel(const amd::Kernel& kernel //!< AMD kernel object - ) const; + inline void AddKernel(const amd::Kernel& kernel //!< AMD kernel object + ) const; //! Adds a dopp desktop texture reference void addDoppRef(const Memory* memory, //!< GPU memory object @@ -414,9 +414,6 @@ class VirtualGPU : public device::VirtualDevice { //! Returns DMA flush management structure const DmaFlushMgmt& dmaFlushMgmt() const { return dmaFlushMgmt_; } - //! Releases GSL memory objects allocated on this queue - void releaseMemObjects(bool scratch = true); - //! Returns the HW ring used on this virtual device uint hwRing() const { return hwRing_; } @@ -525,8 +522,6 @@ class VirtualGPU : public device::VirtualDevice { MemoryRange() : start_(0), end_(0) {} }; - typedef std::map GpuEvents; - //! Finds total amount of necessary iterations inline void findIterations(const amd::NDRangeContainer& sizes, //!< Original workload sizes const amd::NDRange& local, //!< Local workgroup size @@ -552,8 +547,7 @@ class VirtualGPU : public device::VirtualDevice { //! Detects memory dependency for HSAIL kernels and flushes caches bool processMemObjectsHSA(const amd::Kernel& kernel, //!< AMD kernel object for execution const_address params, //!< Pointer to the param's store - bool nativeMem, //!< Native memory objects - std::vector* memList //!< Memory list for KMD tracking + bool nativeMem //!< Native memory objects ); //! Common function for fill memory used by both svm Fill and non-svm fill @@ -586,8 +580,6 @@ class VirtualGPU : public device::VirtualDevice { HwDbgKernelInfo& kernelInfo //!< kernel info for the dispatch ); - GpuEvents gpuEvents_; //!< GPU events - Device& gpuDevice_; //!< physical GPU device amd::Monitor execution_; //!< Lock to serialise access to all device objects uint index_; //!< The virtual device unique index