P4 to Git Change 1465654 by wchau@wchau_OCL_boltzmann on 2017/10/02 16:13:43

SWDEV-120036 - Supporting the cl_amd_device_attribute_query on the ROC device
	- resubmit the codes with the addition of global free memory query support

Affected files ...

... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#66 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#24 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocmemory.cpp#26 edit
... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocsettings.cpp#27 edit


[ROCm/clr commit: 676a9d5003]

projects/clr/rocclr/runtime/device/rocm/rocdevice.cpp

@@ -129,6 +129,7 @@ Device::Device(hsa_agent_t bkendDevice)
     , xferWrite_(nullptr)
     , pro_device_(nullptr)
     , pro_ena_(false)
+    , freeMem_(0)
     , numOfVgpus_(0) {
   group_segment_.handle = 0;
   system_segment_.handle = 0;
@@ -887,6 +888,8 @@ bool Device::populateOCLDeviceConstants() {
     }
   }
 
+  freeMem_ = info_.globalMemSize_;
+
   // Make sure the max allocation size is not larger than the available
   // memory size.
   info_.maxMemAllocSize_ = std::min(info_.maxMemAllocSize_, info_.globalMemSize_);
@@ -1093,26 +1096,35 @@ bool Device::populateOCLDeviceConstants() {
 #endif  // !defined(WITH_LIGHTNING_COMPILER)
   }
 
-  //if (settings().checkExtension(ClAmdDeviceAttributeQuery)) {
-    //info_.simdPerCU_ = deviceInfo_.simdPerCU_;
-    //info_.simdWidth_ = deviceInfo_.simdWidth_;
-    //info_.simdInstructionWidth_ = deviceInfo_.simdInstructionWidth_;
+  if (settings().checkExtension(ClAmdDeviceAttributeQuery)) {
+    info_.simdPerCU_ = deviceInfo_.simdPerCU_;
+    info_.simdWidth_ = deviceInfo_.simdWidth_;
+    info_.simdInstructionWidth_ = deviceInfo_.simdInstructionWidth_;
     if (HSA_STATUS_SUCCESS !=
         hsa_agent_get_info(_bkendDevice, HSA_AGENT_INFO_WAVEFRONT_SIZE, &info_.wavefrontWidth_)) {
       return false;
     }
-    //info_.globalMemChannels_ = palProp.gpuMemoryProperties.performance.vramBusBitWidth / 32;
-    //info_.globalMemChannelBanks_ = 4;
-    //info_.globalMemChannelBankWidth_ = deviceInfo_.memChannelBankWidth_;
-    //info_.localMemSizePerCU_ = deviceInfo_.localMemSizePerCU_;
-    //info_.localMemBanks_ = deviceInfo_.localMemBanks_;
+    if (HSA_STATUS_SUCCESS !=
+        hsa_agent_get_info(_bkendDevice, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MEMORY_WIDTH, &info_.globalMemChannels_)) {
+      return false;
+    }
+    info_.globalMemChannelBanks_ = 4;
+    info_.globalMemChannelBankWidth_ = deviceInfo_.memChannelBankWidth_;
+    info_.localMemSizePerCU_ = deviceInfo_.localMemSizePerCU_;
+    info_.localMemBanks_ = deviceInfo_.localMemBanks_;
     info_.gfxipVersion_ = deviceInfo_.gfxipVersion_;
-    //info_.numAsyncQueues_ = numComputeRings;
-    //info_.numRTQueues_ = numExclusiveComputeRings;
-    //info_.numRTCUs_ = palProp.engineProperties[Pal::EngineTypeExclusiveCompute].maxNumDedicatedCu;
-    //info_.threadTraceEnable_ = settings().threadTraceEnable_;
-  //}
-
+    if (HSA_STATUS_SUCCESS !=
+        hsa_agent_get_info(_bkendDevice, HSA_AGENT_INFO_QUEUES_MAX, &info_.numAsyncQueues_)) {
+      return false;
+    }
+    info_.numRTQueues_ = info_.numAsyncQueues_;
+    if (HSA_STATUS_SUCCESS !=
+        hsa_agent_get_info(_bkendDevice, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT, &info_.numRTCUs_)) {
+      return false;
+    }
+    //TODO: set to true once thread trace support is available
+    info_.threadTraceEnable_ = false;
+  }
 
   return true;
 }
@@ -1136,7 +1148,18 @@ device::VirtualDevice* Device::createVirtualDevice(amd::CommandQueue* queue) {
   return virtualDevice;
 }
 
-bool Device::globalFreeMemory(size_t* freeMemory) const { return false; }
+bool Device::globalFreeMemory(size_t* freeMemory) const {
+  const uint TotalFreeMemory = 0;
+  const uint LargestFreeBlock = 1;
+
+  freeMemory[TotalFreeMemory] = freeMem_ / Ki;
+
+  // since there is no memory heap on ROCm, the biggest free block is
+  // equal to total free local memory
+  freeMemory[LargestFreeBlock] = freeMemory[TotalFreeMemory];
+
+  return true;
+}
 
 bool Device::bindExternalDevice(uint flags, void* const gfxDevice[], void* gfxContext,
                                 bool validateOnly) {
@@ -1399,6 +1422,15 @@ void Device::memFree(void* ptr, size_t size) const {
   }
 }
 
+void Device::updateFreeMemory(size_t size, bool free) {
+  if (free) {
+    freeMem_ += size;
+  }
+  else {
+    freeMem_ -= size;
+  }
+}
+
 void* Device::svmAlloc(amd::Context& context, size_t size, size_t alignment, cl_svm_mem_flags flags,
                        void* svmPtr) const {
   amd::Memory* mem = nullptr;

projects/clr/rocclr/runtime/device/rocm/rocdevice.hpp

@@ -400,6 +400,9 @@ class Device : public NullDevice {
   // Lock protect P2P staging operations
   const std::vector<Memory*>& P2PStages() const { return p2p_stages_; }
 
+  // Update the global free memory size
+  void updateFreeMemory(size_t size, bool free);
+
  private:
   static hsa_ven_amd_loader_1_00_pfn_t amd_loader_ext_table;
 
@@ -433,6 +436,7 @@ class Device : public NullDevice {
   XferBuffers* xferWrite_;  //!< Transfer buffers write
   const IProDevice* pro_device_;  //!< AMDGPUPro device
   bool  pro_ena_;           //!< Extra functionality with AMDGPUPro device, beyond ROCr
+  std::atomic<size_t> freeMem_;   //!< Total of free memory available
 
  public:
   amd::Atomic<uint> numOfVgpus_;  //!< Virtual gpu unique index

projects/clr/rocclr/runtime/device/rocm/rocmemory.cpp

@@ -586,6 +586,7 @@ void Buffer::destroy() {
       }
     } else {
       dev().memFree(deviceMemory_, size());
+      const_cast<Device&>(dev()).updateFreeMemory(size(), true);
     }
   }
 
@@ -669,6 +670,9 @@ bool Buffer::create() {
       deviceMemory_ = dev().hostAlloc(size(), 1, false);
       owner()->setHostMem(deviceMemory_);
     }
+    else {
+      const_cast<Device&>(dev()).updateFreeMemory(size(), false);
+    }
 
     assert(amd::isMultipleOf(deviceMemory_, static_cast<size_t>(dev().info().memBaseAddrAlign_)));
 
@@ -932,6 +936,9 @@ bool Image::create() {
   if (originalDeviceMemory_ == nullptr) {
     originalDeviceMemory_ = dev().hostAlloc(alloc_size, 1, false);
   }
+  else {
+    const_cast<Device&>(dev()).updateFreeMemory(alloc_size, false);
+  }
 
   deviceMemory_ = reinterpret_cast<void*>(
       amd::alignUp(reinterpret_cast<uintptr_t>(originalDeviceMemory_), deviceImageInfo_.alignment));
@@ -1092,6 +1099,7 @@ void Image::destroy() {
 
   if (originalDeviceMemory_ != nullptr) {
     dev().memFree(originalDeviceMemory_, deviceImageInfo_.size);
+    const_cast<Device&>(dev()).updateFreeMemory(size(), true);
   }
 }
 }

projects/clr/rocclr/runtime/device/rocm/rocsettings.cpp

@@ -101,6 +101,8 @@ bool Settings::create(bool fullProfile, int gfxipVersion) {
   if (MesaInterop::Supported()) {
     enableExtension(ClKhrGlSharing);
   }
+  // Enable platform extension
+  enableExtension(ClAmdDeviceAttributeQuery);
 
   // Enable KHR double precision extension
   enableExtension(ClKhrFp64);