SWDEV-528860 - reserve some memory in visible frame buffer (#251)

[ROCm/clr commit: faac50c77a]

projects/clr/rocclr/device/pal/paldevice.cpp

@@ -998,6 +998,23 @@ bool Device::create(Pal::IDevice* device) {
   // Fill the device info structure
   fillDeviceInfo(properties(), heaps_, 16 * Ki, numComputeEngines(), numExclusiveComputeEngines(), iDev());
 
+  // Reserve percentage memory for large frame buffer.
+  // Reserve a threshold size for small frame buffer, used by page table for remote memory mapping
+  Pal::gpusize invisibleSize = heaps_[Pal::GpuHeapInvisible].logicalSize;
+  Pal::gpusize visibleSize = heaps_[Pal::GpuHeapLocal].logicalSize;
+
+  Pal::gpusize maxInvisibleAllocation = std::min((invisibleSize / 100) * 98,
+    invisibleSize < 128 * Mi? 0: invisibleSize - 128 * Mi);
+  Pal::gpusize maxVisibleAllocation = std::min((visibleSize / 100) * 98,
+    visibleSize < 128 * Mi? 0: visibleSize - 128 * Mi);
+
+  if (invisibleSize < visibleSize && invisibleSize > 0) {
+    // Page table is in invisible and its size is smaller. Invisible is the only deciding factor
+    maxFrameBufferAllocation_ = maxInvisibleAllocation;
+  } else {
+    maxFrameBufferAllocation_ = std::max(maxInvisibleAllocation, maxVisibleAllocation);
+  }
+
   if (!ValidateComgr()) {
     LogError("Code object manager initialization failed!");
     return false;

projects/clr/rocclr/device/pal/paldevice.hpp

@@ -675,8 +675,8 @@ class Device : public NullDevice {
   //! Allocates hidden heap for device memory allocations
   void HiddenHeapAlloc(const VirtualGPU& gpu);
 
-  const Pal::GpuMemoryHeapProperties& GetGpuHeapInvisible() const {
-    return heaps_[Pal::GpuHeapInvisible];
+  Pal::gpusize GetMaxFrameBuffer() const {
+    return maxFrameBufferAllocation_;
   }
 
   Pal::gpusize TotalAlloc() const {
@@ -760,6 +760,7 @@ class Device : public NullDevice {
   ICaptureMgr* captureMgr_;                      //!< RGP/UberTrace capture manager
   Pal::GpuMemoryHeapProperties
       heaps_[Pal::GpuHeapCount];         //!< Information about heaps, returned from PAL
+  Pal::gpusize maxFrameBufferAllocation_; //!< To reserve some memory in frame buffer
   std::map<Pal::IQueue*, QueueRecycleInfo*> queue_pool_;  //!< Pool of PAL queues for recycling
   amd::Program* trap_handler_ = nullptr;  //!< Trap handler program for debugger setup
 };

projects/clr/rocclr/device/pal/palmemory.cpp

@@ -116,9 +116,8 @@ bool Memory::create(Resource::MemoryType memType, Resource::CreateParams* params
     }
 
     if (amd::IS_HIP && dev().settings().apuSystem_) {
-      const Pal::GpuMemoryHeapProperties& invisibleHeap = dev().GetGpuHeapInvisible();
       Pal::gpusize totalAlloc = dev().TotalAlloc();
-      if (memType == Local && (totalAlloc > invisibleHeap.logicalSize)) {
+      if (memType == Local && totalAlloc > dev().GetMaxFrameBuffer()) {
         memType = RemoteUSWC;
       }
     }