P4 to Git Change 1576247 by gandryey@gera-w8 on 2018/07/04 16:55:54

SWDEV-158017 - CL_DEVICE_GLOBAL_FREE_MEMORY_AMD doesn't work correctly on PAL backend
	- Change the free memory detection to account possible memory allocation with oversubscription.
	- Make sure resource cache isn't counted as allocated memory

Affected files ...

... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpudevice.cpp#593 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuresource.cpp#244 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/gpu/gpuresource.hpp#88 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.cpp#94 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/paldevice.hpp#31 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.cpp#68 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/pal/palresource.hpp#25 edit

rocclr/runtime/device/gpu/gpudevice.cpp

@@ -1805,7 +1805,8 @@ bool Device::globalFreeMemory(size_t* freeMemory) const {
 
   // Fill free memory info
   freeMemory[TotalFreeMemory] =
-      (memInfo.cardMemAvailableBytes + memInfo.cardExtMemAvailableBytes) / Ki;
+      (memInfo.cardMemAvailableBytes + memInfo.cardExtMemAvailableBytes +   
+       resourceCache().lclCacheSize()) / Ki;
   freeMemory[LargestFreeBlock] =
       std::max(memInfo.cardLargestFreeBlockBytes, memInfo.cardExtLargestFreeBlockBytes) / Ki;
   if (settings().apuSystem_) {

rocclr/runtime/device/gpu/gpuresource.cpp

@@ -1748,6 +1748,9 @@ bool ResourceCache::addCalResource(Resource::CalResourceDesc* desc, GslResourceR
       // Add the current resource to the cache
       resCache_.push_front({descCached, ref});
       cacheSize_ += size;
+      if (desc->type_ == Resource::Local) {
+          lclCacheSize_ += size;
+      }
       result = true;
     }
   }
@@ -1777,10 +1780,13 @@ GslResourceReference* ResourceCache::findCalResource(Resource::CalResourceDesc*
         (entry->mipLevels_ == desc->mipLevels_) && (entry->isAllocSVM_ == desc->isAllocSVM_) &&
         (entry->isAllocExecute_ == desc->isAllocExecute_)) {
       ref = it.second;
+      cacheSize_ -= size;
+      if (entry->type_ == Resource::Local) {
+          lclCacheSize_ -= size;
+      }
       delete it.first;
       // Remove the found etry from the cache
       resCache_.remove(it);
-      cacheSize_ -= size;
       break;
     }
   }
@@ -1823,12 +1829,16 @@ void ResourceCache::removeLast() {
 
   size_t size = getResourceSize(entry.first);
 
+  cacheSize_ -= size;
+  if (entry.first->type_ == Resource::Local) {
+      lclCacheSize_ -= size;
+  }
+
   // Delete CalResourceDesc
   delete entry.first;
 
   // Destroy GSL resource
   entry.second->release();
-  cacheSize_ -= size;
 }
 
 }  // namespace gpu

rocclr/runtime/device/gpu/gpuresource.hpp

@@ -428,7 +428,10 @@ class ResourceCache : public amd::HeapObject {
  public:
   //! Default constructor
   ResourceCache(size_t cacheSizeLimit)
-      : lockCacheOps_("CAL resource cache", true), cacheSize_(0), cacheSizeLimit_(cacheSizeLimit) {}
+      : lockCacheOps_("CAL resource cache", true)
+      , cacheSize_(0)
+      , lclCacheSize_(0)
+      , cacheSizeLimit_(cacheSizeLimit) {}
 
   //! Default destructor
   ~ResourceCache();
@@ -446,6 +449,9 @@ class ResourceCache : public amd::HeapObject {
   //! Destroys cache
   bool free(size_t minCacheEntries = 0);
 
+  //! Returns the size of local memory, stored in the cache
+  size_t lclCacheSize() const { return lclCacheSize_; }
+
  private:
   //! Disable copy constructor
   ResourceCache(const ResourceCache&);
@@ -462,6 +468,7 @@ class ResourceCache : public amd::HeapObject {
   amd::Monitor lockCacheOps_;  //!< Lock to serialise cache access
 
   size_t cacheSize_;       //!< Current cache size in bytes
+  size_t lclCacheSize_;    //!< Local memory stored in the cache
   size_t cacheSizeLimit_;  //!< Cache size limit in bytes
 
   //! CAL resource cache

rocclr/runtime/device/pal/paldevice.cpp

@@ -846,15 +846,14 @@ bool Device::create(Pal::IDevice* device) {
   // Commit the new settings for the device
   result = iDev()->CommitSettingsAndInit();
 
-  Pal::GpuMemoryHeapProperties heaps[Pal::GpuHeapCount];
-  iDev()->GetGpuMemoryHeapProperties(heaps);
+  iDev()->GetGpuMemoryHeapProperties(heaps_);
 
   Pal::WorkStationCaps wscaps = {};
   iDev()->QueryWorkStationCaps(&wscaps);
 
   pal::Settings* gpuSettings = reinterpret_cast<pal::Settings*>(settings_);
   if ((gpuSettings == nullptr) ||
-      !gpuSettings->create(properties(), heaps, wscaps, appProfile_.reportAsOCL12Device())) {
+      !gpuSettings->create(properties(), heaps_, wscaps, appProfile_.reportAsOCL12Device())) {
     return false;
   }
   numComputeEngines_ = std::min(numComputeEngines_, settings().numComputeRings_);
@@ -885,7 +884,7 @@ bool Device::create(Pal::IDevice* device) {
   }
 
   // Fill the device info structure
-  fillDeviceInfo(properties(), heaps, 16 * Ki, numComputeEngines(), numExclusiveComputeEngines());
+  fillDeviceInfo(properties(), heaps_, 16 * Ki, numComputeEngines(), numExclusiveComputeEngines());
 
 #ifdef DEBUG
   std::stringstream message;
@@ -901,7 +900,7 @@ bool Device::create(Pal::IDevice* device) {
 #endif  // DEBUG
 
   for (uint i = 0; i < Pal::GpuHeap::GpuHeapCount; ++i) {
-    freeMem[i] = heaps[i].heapSize;
+    allocedMem[i] = 0;
   }
 
   // Allocate SRD manager
@@ -1677,15 +1676,31 @@ bool Device::globalFreeMemory(size_t* freeMemory) const {
     return false;
   }
 
-  Pal::gpusize local = freeMem[Pal::GpuHeapLocal];
-  Pal::gpusize invisible = freeMem[Pal::GpuHeapInvisible];
+  Pal::gpusize local = allocedMem[Pal::GpuHeapLocal];
+  Pal::gpusize invisible = allocedMem[Pal::GpuHeapInvisible] - resourceCache().lclCacheSize();
+  // Calculate free memory
+  if (local >= heaps_[Pal::GpuHeapLocal].heapSize) {
+    local = 0;
+  } else {
+    local = heaps_[Pal::GpuHeapLocal].heapSize - local;
+  }
+  if (invisible >= info().maxMemAllocSize_) {
+    invisible = 0;
+  } else {
+    invisible = info().maxMemAllocSize_ - invisible;
+  }
 
   // Fill free memory info
   freeMemory[TotalFreeMemory] = static_cast<size_t>((local + invisible) / Ki);
   freeMemory[LargestFreeBlock] = static_cast<size_t>(std::max(local, invisible) / Ki);
 
   if (settings().apuSystem_) {
-    Pal::gpusize uswc = freeMem[Pal::GpuHeapGartUswc];
+    Pal::gpusize uswc = allocedMem[Pal::GpuHeapGartUswc];
+    if (uswc >= heaps_[Pal::GpuHeapGartUswc].heapSize) {
+      uswc = 0;
+    } else {
+      uswc = heaps_[Pal::GpuHeapGartUswc].heapSize - uswc;
+    }
     uswc /= Ki;
     freeMemory[TotalFreeMemory] += static_cast<size_t>(uswc);
     if (freeMemory[LargestFreeBlock] < uswc) {
@@ -2124,11 +2139,11 @@ void Device::SrdManager::freeSrdSlot(uint64_t addr) {
   assert(false && "Wrong slot address!");
 }
 
-void Device::updateFreeMemory(Pal::GpuHeap heap, Pal::gpusize size, bool free) {
+void Device::updateAllocedMemory(Pal::GpuHeap heap, Pal::gpusize size, bool free) const {
   if (free) {
-    freeMem[heap] += size;
-  } else {
-    freeMem[heap] -= size;
+    allocedMem[heap] -= size;
+  }  else {
+    allocedMem[heap] += size;
   }
 }
 

rocclr/runtime/device/pal/paldevice.hpp

@@ -468,10 +468,10 @@ class Device : public NullDevice {
   RgpCaptureMgr* rgpCaptureMgr() const { return rgpCaptureMgr_; }
 
   //! Update free memory for OCL extension
-  void updateFreeMemory(Pal::GpuHeap heap,  //!< PAL GPU heap for update
-                        Pal::gpusize size,  //!< Size of alocated/destroyed memory
-                        bool free           //!< TRUE if runtime frees memory
-                        );
+  void updateAllocedMemory(Pal::GpuHeap heap,  //!< PAL GPU heap for update
+                           Pal::gpusize size,  //!< Size of alocated/destroyed memory
+                           bool free           //!< TRUE if runtime frees memory
+                           ) const;
 
   //! Create internal blit program
   bool createBlitProgram();
@@ -594,9 +594,10 @@ class Device : public NullDevice {
   mutable bool freeCPUMem_;              //!< flag to mark GPU free SVM CPU mem
   Pal::DeviceProperties properties_;     //!< PAL device properties
   Pal::IDevice* device_;                 //!< PAL device object
-  std::atomic<Pal::gpusize> freeMem[Pal::GpuHeap::GpuHeapCount];  //!< Free memory counter
+  mutable std::atomic<Pal::gpusize> allocedMem[Pal::GpuHeap::GpuHeapCount];  //!< Free memory counter
   std::unordered_set<Resource*>* resourceList_;   //!< Active resource list
   RgpCaptureMgr*   rgpCaptureMgr_;       //!< RGP capture manager
+  Pal::GpuMemoryHeapProperties heaps_[Pal::GpuHeapCount]; //!< Information about heaps, returned from PAL
 };
 
 /*@}*/} // namespace pal

rocclr/runtime/device/pal/palresource.cpp

@@ -51,8 +51,10 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
       return nullptr;
     }
   }
-  // Update free memory size counters
-  const_cast<Device&>(dev).updateFreeMemory(createInfo.heaps[0], createInfo.size, false);
+  if (!createInfo.flags.sdiExternal) {
+    // Update free memory size counters
+    dev.updateAllocedMemory(createInfo.heaps[0], createInfo.size, false);
+  }
   return memRef;
 }
 
@@ -75,7 +77,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
     }
   }
   // Update free memory size counters
-  const_cast<Device&>(dev).updateFreeMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size, false);
+  dev.updateAllocedMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size, false);
   return memRef;
 }
 
@@ -97,8 +99,7 @@ GpuMemoryReference* GpuMemoryReference::Create(const Device& dev,
     }
   }
   // Update free memory size counters
-  const_cast<Device&>(dev).updateFreeMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size,
-                                            false);
+  dev.updateAllocedMemory(Pal::GpuHeap::GpuHeapGartCacheable, createInfo.size, false);
   return memRef;
 }
 
@@ -180,6 +181,12 @@ GpuMemoryReference::~GpuMemoryReference() {
     iMem()->Unmap();
   }
   if (0 != iMem()) {
+    if (!(iMem()->Desc().flags.isShared ||
+          iMem()->Desc().flags.isExternal ||
+          iMem()->Desc().flags.isExternPhys)) {
+      // Update free memory size counters
+      device_.updateAllocedMemory(iMem()->Desc().preferredHeap, iMem()->Desc().size, true);
+    }
     iMem()->Destroy();
     gpuMem_ = nullptr;
   }
@@ -288,34 +295,6 @@ Resource::Resource(const Device& gpuDev, size_t width, size_t height, size_t dep
 
 // ================================================================================================
 Resource::~Resource() {
-  Pal::GpuHeap heap = Pal::GpuHeapCount;
-  switch (memoryType()) {
-    case Persistent:
-      heap = Pal::GpuHeapLocal;
-      break;
-    case RemoteUSWC:
-      heap = Pal::GpuHeapGartUswc;
-      break;
-    case Pinned:
-    case Remote:
-      heap = Pal::GpuHeapGartCacheable;
-      break;
-    case Shader:
-    case BusAddressable:
-    case ExternalPhysical:
-    // Fall through to process the memory allocation ...
-    case Local:
-      heap = Pal::GpuHeapInvisible;
-      break;
-    default:
-      heap = Pal::GpuHeapLocal;
-      break;
-  }
-  if ((memRef_ != nullptr) && (heap != Pal::GpuHeapCount)) {
-    // Update free memory size counters
-    const_cast<Device&>(dev()).updateFreeMemory(heap, iMem()->Desc().size, true);
-  }
-
   free();
 
   if ((nullptr != image_) &&
@@ -1228,6 +1207,7 @@ void Resource::free()
 
     // Add resource to the cache
     if (!dev().resourceCache().addGpuMemory(&desc_, memRef_, subOffset_)) {
+      // Free PAL resource
       palFree();
     }
   }
@@ -2015,6 +1995,9 @@ bool ResourceCache::addGpuMemory(Resource::Descriptor* desc,
       resCache_.push_front({descCached, ref});
       ref->gpu_ = nullptr;
       cacheSize_ += size;
+      if (desc->type_ == Resource::Local) {
+        lclCacheSize_ += size;
+      }
       result = true;
     }
   }
@@ -2056,10 +2039,13 @@ GpuMemoryReference* ResourceCache::findGpuMemory(Resource::Descriptor* desc, Pal
         (size > (sizeRes >> 2)) && ((it.second->iMem()->Desc().gpuVirtAddr % alignment) == 0) &&
         (entry->isAllocExecute_ == desc->isAllocExecute_)) {
       ref = it.second;
+      cacheSize_ -= sizeRes;
+      if (entry->type_ == Resource::Local) {
+          lclCacheSize_ -= sizeRes;
+      }
       delete it.first;
       // Remove the found etry from the cache
       resCache_.remove(it);
-      cacheSize_ -= sizeRes;
       break;
     }
   }
@@ -2090,9 +2076,12 @@ void ResourceCache::removeLast()
     amd::ScopedLock l(&lockCacheOps_);
     entry = resCache_.back();
     resCache_.pop_back();
+    cacheSize_ -= entry.second->iMem()->Desc().size;
+    if (entry.first->type_ == Resource::Local) {
+      lclCacheSize_ -= entry.second->iMem()->Desc().size;
+    }
     // Delete Descriptor
     delete entry.first;
-    cacheSize_ -= entry.second->iMem()->Desc().size;
   }
 
   // Destroy PAL resource

rocclr/runtime/device/pal/palresource.hpp

@@ -531,6 +531,7 @@ class ResourceCache : public amd::HeapObject {
   ResourceCache(Device* device, size_t cacheSizeLimit)
       : lockCacheOps_("PAL resource cache", true)
       , cacheSize_(0)
+      , lclCacheSize_(0)
       , cacheSizeLimit_(cacheSizeLimit)
       , mem_sub_alloc_local_(device)
       , mem_sub_alloc_coarse_ (device)
@@ -557,6 +558,9 @@ class ResourceCache : public amd::HeapObject {
   //! Returns true if cache was freed and false if cache is already empty.
   bool free(size_t minCacheEntries = 0);
 
+  //! Returns the size of local memory, stored in the cache
+  size_t lclCacheSize() const { return lclCacheSize_; }
+
  private:
   //! Disable copy constructor
   ResourceCache(const ResourceCache&);
@@ -570,6 +574,7 @@ class ResourceCache : public amd::HeapObject {
   amd::Monitor lockCacheOps_;  //!< Lock to serialise cache access
 
   size_t cacheSize_;            //!< Current cache size in bytes
+  size_t lclCacheSize_;         //!< Local memory stored in the cache
   const size_t cacheSizeLimit_; //!< Cache size limit in bytes
 
   //! PAL resource cache