From 7e185b873e2804bf6c9dc1e317385e1091377b22 Mon Sep 17 00:00:00 2001 From: foreman Date: Wed, 4 Jul 2018 17:04:53 -0400 Subject: [PATCH] P4 to Git Change 1576247 by gandryey@gera-w8 on 2018/07/04 16:55:54 SWDEV-158017 - CL_DEVICE_GLOBAL_FREE_MEMORY_AMD doesn't work correctly on PAL backend - Change the free memory detection to account possible memory allocation with oversubscription. - Make sure resource cache isn't counted as allocated memory Affected files ... ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpudevice.cpp#593 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuresource.cpp#244 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuresource.hpp#88 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#94 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.hpp#31 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.cpp#68 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.hpp#25 edit --- rocclr/runtime/device/gpu/gpudevice.cpp | 3 +- rocclr/runtime/device/gpu/gpuresource.cpp | 14 +++++- rocclr/runtime/device/gpu/gpuresource.hpp | 9 +++- rocclr/runtime/device/pal/paldevice.cpp | 39 ++++++++++----- rocclr/runtime/device/pal/paldevice.hpp | 11 +++-- rocclr/runtime/device/pal/palresource.cpp | 59 +++++++++-------------- rocclr/runtime/device/pal/palresource.hpp | 5 ++ 7 files changed, 84 insertions(+), 56 deletions(-) diff --git a/rocclr/runtime/device/gpu/gpudevice.cpp b/rocclr/runtime/device/gpu/gpudevice.cpp index 7e43ce3ac5..0ac9a5ae68 100644 --- a/rocclr/runtime/device/gpu/gpudevice.cpp +++ b/rocclr/runtime/device/gpu/gpudevice.cpp @@ -1805,7 +1805,8 @@ bool Device::globalFreeMemory(size_t* freeMemory) const { // Fill free memory info freeMemory[TotalFreeMemory] = - (memInfo.cardMemAvailableBytes + memInfo.cardExtMemAvailableBytes) / Ki; + (memInfo.cardMemAvailableBytes + memInfo.cardExtMemAvailableBytes + + resourceCache().lclCacheSize()) / Ki; freeMemory[LargestFreeBlock] = std::max(memInfo.cardLargestFreeBlockBytes, memInfo.cardExtLargestFreeBlockBytes) / Ki; if (settings().apuSystem_) { diff --git a/rocclr/runtime/device/gpu/gpuresource.cpp b/rocclr/runtime/device/gpu/gpuresource.cpp index 8c1ddcfd9a..bdf8c28cbd 100644 --- a/rocclr/runtime/device/gpu/gpuresource.cpp +++ b/rocclr/runtime/device/gpu/gpuresource.cpp @@ -1748,6 +1748,9 @@ bool ResourceCache::addCalResource(Resource::CalResourceDesc* desc, GslResourceR // Add the current resource to the cache resCache_.push_front({descCached, ref}); cacheSize_ += size; + if (desc->type_ == Resource::Local) { + lclCacheSize_ += size; + } result = true; } } @@ -1777,10 +1780,13 @@ GslResourceReference* ResourceCache::findCalResource(Resource::CalResourceDesc* (entry->mipLevels_ == desc->mipLevels_) && (entry->isAllocSVM_ == desc->isAllocSVM_) && (entry->isAllocExecute_ == desc->isAllocExecute_)) { ref = it.second; + cacheSize_ -= size; + if (entry->type_ == Resource::Local) { + lclCacheSize_ -= size; + } delete it.first; // Remove the found etry from the cache resCache_.remove(it); - cacheSize_ -= size; break; } } @@ -1823,12 +1829,16 @@ void ResourceCache::removeLast() { size_t size = getResourceSize(entry.first); + cacheSize_ -= size; + if (entry.first->type_ == Resource::Local) { + lclCacheSize_ -= size; + } + // Delete CalResourceDesc delete entry.first; // Destroy GSL resource entry.second->release(); - cacheSize_ -= size; } } // namespace gpu diff --git a/rocclr/runtime/device/gpu/gpuresource.hpp b/rocclr/runtime/device/gpu/gpuresource.hpp index 251ec2b8ee..350c930de3 100644 --- a/rocclr/runtime/device/gpu/gpuresource.hpp +++ b/rocclr/runtime/device/gpu/gpuresource.hpp @@ -428,7 +428,10 @@ class ResourceCache : public amd::HeapObject { public: //! Default constructor ResourceCache(size_t cacheSizeLimit) - : lockCacheOps_("CAL resource cache", true), cacheSize_(0), cacheSizeLimit_(cacheSizeLimit) {} + : lockCacheOps_("CAL resource cache", true) + , cacheSize_(0) + , lclCacheSize_(0) + , cacheSizeLimit_(cacheSizeLimit) {} //! Default destructor ~ResourceCache(); @@ -446,6 +449,9 @@ class ResourceCache : public amd::HeapObject { //! Destroys cache bool free(size_t minCacheEntries = 0); + //! Returns the size of local memory, stored in the cache + size_t lclCacheSize() const { return lclCacheSize_; } + private: //! Disable copy constructor ResourceCache(const ResourceCache&); @@ -462,6 +468,7 @@ class ResourceCache : public amd::HeapObject { amd::Monitor lockCacheOps_; //!< Lock to serialise cache access size_t cacheSize_; //!< Current cache size in bytes + size_t lclCacheSize_; //!< Local memory stored in the cache size_t cacheSizeLimit_; //!< Cache size limit in bytes //! CAL resource cache diff --git a/rocclr/runtime/device/pal/paldevice.cpp b/rocclr/runtime/device/pal/paldevice.cpp index 5a9e528321..9c31edc6a5 100644 --- a/rocclr/runtime/device/pal/paldevice.cpp +++ b/rocclr/runtime/device/pal/paldevice.cpp @@ -846,15 +846,14 @@ bool Device::create(Pal::IDevice* device) { // Commit the new settings for the device result = iDev()->CommitSettingsAndInit(); - Pal::GpuMemoryHeapProperties heaps[Pal::GpuHeapCount]; - iDev()->GetGpuMemoryHeapProperties(heaps); + iDev()->GetGpuMemoryHeapProperties(heaps_); Pal::WorkStationCaps wscaps = {}; iDev()->QueryWorkStationCaps(&wscaps); pal::Settings* gpuSettings = reinterpret_cast(settings_); if ((gpuSettings == nullptr) || - !gpuSettings->create(properties(), heaps, wscaps, appProfile_.reportAsOCL12Device())) { + !gpuSettings->create(properties(), heaps_, wscaps, appProfile_.reportAsOCL12Device())) { return false; } numComputeEngines_ = std::min(numComputeEngines_, settings().numComputeRings_); @@ -885,7 +884,7 @@ bool Device::create(Pal::IDevice* device) { } // Fill the device info structure - fillDeviceInfo(properties(), heaps, 16 * Ki, numComputeEngines(), numExclusiveComputeEngines()); + fillDeviceInfo(properties(), heaps_, 16 * Ki, numComputeEngines(), numExclusiveComputeEngines()); #ifdef DEBUG std::stringstream message; @@ -901,7 +900,7 @@ bool Device::create(Pal::IDevice* device) { #endif // DEBUG for (uint i = 0; i < Pal::GpuHeap::GpuHeapCount; ++i) { - freeMem[i] = heaps[i].heapSize; + allocedMem[i] = 0; } // Allocate SRD manager @@ -1677,15 +1676,31 @@ bool Device::globalFreeMemory(size_t* freeMemory) const { return false; } - Pal::gpusize local = freeMem[Pal::GpuHeapLocal]; - Pal::gpusize invisible = freeMem[Pal::GpuHeapInvisible]; + Pal::gpusize local = allocedMem[Pal::GpuHeapLocal]; + Pal::gpusize invisible = allocedMem[Pal::GpuHeapInvisible] - resourceCache().lclCacheSize(); + // Calculate free memory + if (local >= heaps_[Pal::GpuHeapLocal].heapSize) { + local = 0; + } else { + local = heaps_[Pal::GpuHeapLocal].heapSize - local; + } + if (invisible >= info().maxMemAllocSize_) { + invisible = 0; + } else { + invisible = info().maxMemAllocSize_ - invisible; + } // Fill free memory info freeMemory[TotalFreeMemory] = static_cast((local + invisible) / Ki); freeMemory[LargestFreeBlock] = static_cast(std::max(local, invisible) / Ki); if (settings().apuSystem_) { - Pal::gpusize uswc = freeMem[Pal::GpuHeapGartUswc]; + Pal::gpusize uswc = allocedMem[Pal::GpuHeapGartUswc]; + if (uswc >= heaps_[Pal::GpuHeapGartUswc].heapSize) { + uswc = 0; + } else { + uswc = heaps_[Pal::GpuHeapGartUswc].heapSize - uswc; + } uswc /= Ki; freeMemory[TotalFreeMemory] += static_cast(uswc); if (freeMemory[LargestFreeBlock] < uswc) { @@ -2124,11 +2139,11 @@ void Device::SrdManager::freeSrdSlot(uint64_t addr) { assert(false && "Wrong slot address!"); } -void Device::updateFreeMemory(Pal::GpuHeap heap, Pal::gpusize size, bool free) { +void Device::updateAllocedMemory(Pal::GpuHeap heap, Pal::gpusize size, bool free) const { if (free) { - freeMem[heap] += size; - } else { - freeMem[heap] -= size; + allocedMem[heap] -= size; + } else { + allocedMem[heap] += size; } } diff --git a/rocclr/runtime/device/pal/paldevice.hpp b/rocclr/runtime/device/pal/paldevice.hpp index 1b5e1b3b59..50a7aaf4cc 100644 --- a/rocclr/runtime/device/pal/paldevice.hpp +++ b/rocclr/runtime/device/pal/paldevice.hpp @@ -468,10 +468,10 @@ class Device : public NullDevice { RgpCaptureMgr* rgpCaptureMgr() const { return rgpCaptureMgr_; } //! Update free memory for OCL extension - void updateFreeMemory(Pal::GpuHeap heap, //!< PAL GPU heap for update - Pal::gpusize size, //!< Size of alocated/destroyed memory - bool free //!< TRUE if runtime frees memory - ); + void updateAllocedMemory(Pal::GpuHeap heap, //!< PAL GPU heap for update + Pal::gpusize size, //!< Size of alocated/destroyed memory + bool free //!< TRUE if runtime frees memory + ) const; //! Create internal blit program bool createBlitProgram(); @@ -594,9 +594,10 @@ class Device : public NullDevice { mutable bool freeCPUMem_; //!< flag to mark GPU free SVM CPU mem Pal::DeviceProperties properties_; //!< PAL device properties Pal::IDevice* device_; //!< PAL device object - std::atomic freeMem[Pal::GpuHeap::GpuHeapCount]; //!< Free memory counter + mutable std::atomic allocedMem[Pal::GpuHeap::GpuHeapCount]; //!< Free memory counter std::unordered_set* resourceList_; //!< Active resource list RgpCaptureMgr* rgpCaptureMgr_; //!< RGP capture manager + Pal::GpuMemoryHeapProperties heaps_[Pal::GpuHeapCount]; //!< Information about heaps, returned from PAL }; /*@}*/} // namespace pal diff --git a/rocclr/runtime/device/pal/palresource.cpp b/rocclr/runtime/device/pal/palresource.cpp index 63a2e15e99..9df1a955b6 100644 --- a/rocclr/runtime/device/pal/palresource.cpp +++ b/rocclr/runtime/device/pal/palresource.cpp @@ -51,8 +51,10 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, return nullptr; } } - // Update free memory size counters - const_cast(dev).updateFreeMemory(createInfo.heaps[0], createInfo.size, false); + if (!createInfo.flags.sdiExternal) { + // Update free memory size counters + dev.updateAllocedMemory(createInfo.heaps[0], createInfo.size, false); + } return memRef; } @@ -75,7 +77,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, } } // Update free memory size counters - const_cast(dev).updateFreeMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size, false); + dev.updateAllocedMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size, false); return memRef; } @@ -97,8 +99,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev, } } // Update free memory size counters - const_cast(dev).updateFreeMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size, - false); + dev.updateAllocedMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size, false); return memRef; } @@ -180,6 +181,12 @@ GpuMemoryReference::~GpuMemoryReference() { iMem()->Unmap(); } if (0 != iMem()) { + if (!(iMem()->Desc().flags.isShared || + iMem()->Desc().flags.isExternal || + iMem()->Desc().flags.isExternPhys)) { + // Update free memory size counters + device_.updateAllocedMemory(iMem()->Desc().preferredHeap, iMem()->Desc().size, true); + } iMem()->Destroy(); gpuMem_ = nullptr; } @@ -288,34 +295,6 @@ Resource::Resource(const Device& gpuDev, size_t width, size_t height, size_t dep // ================================================================================================ Resource::~Resource() { - Pal::GpuHeap heap = Pal::GpuHeapCount; - switch (memoryType()) { - case Persistent: - heap = Pal::GpuHeapLocal; - break; - case RemoteUSWC: - heap = Pal::GpuHeapGartUswc; - break; - case Pinned: - case Remote: - heap = Pal::GpuHeapGartCacheable; - break; - case Shader: - case BusAddressable: - case ExternalPhysical: - // Fall through to process the memory allocation ... - case Local: - heap = Pal::GpuHeapInvisible; - break; - default: - heap = Pal::GpuHeapLocal; - break; - } - if ((memRef_ != nullptr) && (heap != Pal::GpuHeapCount)) { - // Update free memory size counters - const_cast(dev()).updateFreeMemory(heap, iMem()->Desc().size, true); - } - free(); if ((nullptr != image_) && @@ -1228,6 +1207,7 @@ void Resource::free() // Add resource to the cache if (!dev().resourceCache().addGpuMemory(&desc_, memRef_, subOffset_)) { + // Free PAL resource palFree(); } } @@ -2015,6 +1995,9 @@ bool ResourceCache::addGpuMemory(Resource::Descriptor* desc, resCache_.push_front({descCached, ref}); ref->gpu_ = nullptr; cacheSize_ += size; + if (desc->type_ == Resource::Local) { + lclCacheSize_ += size; + } result = true; } } @@ -2056,10 +2039,13 @@ GpuMemoryReference* ResourceCache::findGpuMemory(Resource::Descriptor* desc, Pal (size > (sizeRes >> 2)) && ((it.second->iMem()->Desc().gpuVirtAddr % alignment) == 0) && (entry->isAllocExecute_ == desc->isAllocExecute_)) { ref = it.second; + cacheSize_ -= sizeRes; + if (entry->type_ == Resource::Local) { + lclCacheSize_ -= sizeRes; + } delete it.first; // Remove the found etry from the cache resCache_.remove(it); - cacheSize_ -= sizeRes; break; } } @@ -2090,9 +2076,12 @@ void ResourceCache::removeLast() amd::ScopedLock l(&lockCacheOps_); entry = resCache_.back(); resCache_.pop_back(); + cacheSize_ -= entry.second->iMem()->Desc().size; + if (entry.first->type_ == Resource::Local) { + lclCacheSize_ -= entry.second->iMem()->Desc().size; + } // Delete Descriptor delete entry.first; - cacheSize_ -= entry.second->iMem()->Desc().size; } // Destroy PAL resource diff --git a/rocclr/runtime/device/pal/palresource.hpp b/rocclr/runtime/device/pal/palresource.hpp index 96f55c6f10..499993eb65 100644 --- a/rocclr/runtime/device/pal/palresource.hpp +++ b/rocclr/runtime/device/pal/palresource.hpp @@ -531,6 +531,7 @@ class ResourceCache : public amd::HeapObject { ResourceCache(Device* device, size_t cacheSizeLimit) : lockCacheOps_("PAL resource cache", true) , cacheSize_(0) + , lclCacheSize_(0) , cacheSizeLimit_(cacheSizeLimit) , mem_sub_alloc_local_(device) , mem_sub_alloc_coarse_ (device) @@ -557,6 +558,9 @@ class ResourceCache : public amd::HeapObject { //! Returns true if cache was freed and false if cache is already empty. bool free(size_t minCacheEntries = 0); + //! Returns the size of local memory, stored in the cache + size_t lclCacheSize() const { return lclCacheSize_; } + private: //! Disable copy constructor ResourceCache(const ResourceCache&); @@ -570,6 +574,7 @@ class ResourceCache : public amd::HeapObject { amd::Monitor lockCacheOps_; //!< Lock to serialise cache access size_t cacheSize_; //!< Current cache size in bytes + size_t lclCacheSize_; //!< Local memory stored in the cache const size_t cacheSizeLimit_; //!< Cache size limit in bytes //! PAL resource cache