From 8234cb0ac1f3211973f923a66a5d5a7c69219aad Mon Sep 17 00:00:00 2001 From: foreman Date: Mon, 2 Oct 2017 16:31:08 -0400 Subject: [PATCH] P4 to Git Change 1465654 by wchau@wchau_OCL_boltzmann on 2017/10/02 16:13:43 SWDEV-120036 - Supporting the cl_amd_device_attribute_query on the ROC device - resubmit the codes with the addition of global free memory query support Affected files ... ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.cpp#66 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocdevice.hpp#24 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocmemory.cpp#26 edit ... //depot/stg/opencl/drivers/opencl/runtime/device/rocm/rocsettings.cpp#27 edit [ROCm/clr commit: 676a9d50033629357b5a252d6db06480cfab571b] --- .../rocclr/runtime/device/rocm/rocdevice.cpp | 64 ++++++++++++++----- .../rocclr/runtime/device/rocm/rocdevice.hpp | 4 ++ .../rocclr/runtime/device/rocm/rocmemory.cpp | 8 +++ .../runtime/device/rocm/rocsettings.cpp | 2 + 4 files changed, 62 insertions(+), 16 deletions(-) diff --git a/projects/clr/rocclr/runtime/device/rocm/rocdevice.cpp b/projects/clr/rocclr/runtime/device/rocm/rocdevice.cpp index c7052400c9..8deb1789cf 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rocdevice.cpp +++ b/projects/clr/rocclr/runtime/device/rocm/rocdevice.cpp @@ -129,6 +129,7 @@ Device::Device(hsa_agent_t bkendDevice) , xferWrite_(nullptr) , pro_device_(nullptr) , pro_ena_(false) + , freeMem_(0) , numOfVgpus_(0) { group_segment_.handle = 0; system_segment_.handle = 0; @@ -887,6 +888,8 @@ bool Device::populateOCLDeviceConstants() { } } + freeMem_ = info_.globalMemSize_; + // Make sure the max allocation size is not larger than the available // memory size. info_.maxMemAllocSize_ = std::min(info_.maxMemAllocSize_, info_.globalMemSize_); @@ -1093,26 +1096,35 @@ bool Device::populateOCLDeviceConstants() { #endif // !defined(WITH_LIGHTNING_COMPILER) } - //if (settings().checkExtension(ClAmdDeviceAttributeQuery)) { - //info_.simdPerCU_ = deviceInfo_.simdPerCU_; - //info_.simdWidth_ = deviceInfo_.simdWidth_; - //info_.simdInstructionWidth_ = deviceInfo_.simdInstructionWidth_; + if (settings().checkExtension(ClAmdDeviceAttributeQuery)) { + info_.simdPerCU_ = deviceInfo_.simdPerCU_; + info_.simdWidth_ = deviceInfo_.simdWidth_; + info_.simdInstructionWidth_ = deviceInfo_.simdInstructionWidth_; if (HSA_STATUS_SUCCESS != hsa_agent_get_info(_bkendDevice, HSA_AGENT_INFO_WAVEFRONT_SIZE, &info_.wavefrontWidth_)) { return false; } - //info_.globalMemChannels_ = palProp.gpuMemoryProperties.performance.vramBusBitWidth / 32; - //info_.globalMemChannelBanks_ = 4; - //info_.globalMemChannelBankWidth_ = deviceInfo_.memChannelBankWidth_; - //info_.localMemSizePerCU_ = deviceInfo_.localMemSizePerCU_; - //info_.localMemBanks_ = deviceInfo_.localMemBanks_; + if (HSA_STATUS_SUCCESS != + hsa_agent_get_info(_bkendDevice, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_MEMORY_WIDTH, &info_.globalMemChannels_)) { + return false; + } + info_.globalMemChannelBanks_ = 4; + info_.globalMemChannelBankWidth_ = deviceInfo_.memChannelBankWidth_; + info_.localMemSizePerCU_ = deviceInfo_.localMemSizePerCU_; + info_.localMemBanks_ = deviceInfo_.localMemBanks_; info_.gfxipVersion_ = deviceInfo_.gfxipVersion_; - //info_.numAsyncQueues_ = numComputeRings; - //info_.numRTQueues_ = numExclusiveComputeRings; - //info_.numRTCUs_ = palProp.engineProperties[Pal::EngineTypeExclusiveCompute].maxNumDedicatedCu; - //info_.threadTraceEnable_ = settings().threadTraceEnable_; - //} - + if (HSA_STATUS_SUCCESS != + hsa_agent_get_info(_bkendDevice, HSA_AGENT_INFO_QUEUES_MAX, &info_.numAsyncQueues_)) { + return false; + } + info_.numRTQueues_ = info_.numAsyncQueues_; + if (HSA_STATUS_SUCCESS != + hsa_agent_get_info(_bkendDevice, (hsa_agent_info_t)HSA_AMD_AGENT_INFO_COMPUTE_UNIT_COUNT, &info_.numRTCUs_)) { + return false; + } + //TODO: set to true once thread trace support is available + info_.threadTraceEnable_ = false; + } return true; } @@ -1136,7 +1148,18 @@ device::VirtualDevice* Device::createVirtualDevice(amd::CommandQueue* queue) { return virtualDevice; } -bool Device::globalFreeMemory(size_t* freeMemory) const { return false; } +bool Device::globalFreeMemory(size_t* freeMemory) const { + const uint TotalFreeMemory = 0; + const uint LargestFreeBlock = 1; + + freeMemory[TotalFreeMemory] = freeMem_ / Ki; + + // since there is no memory heap on ROCm, the biggest free block is + // equal to total free local memory + freeMemory[LargestFreeBlock] = freeMemory[TotalFreeMemory]; + + return true; +} bool Device::bindExternalDevice(uint flags, void* const gfxDevice[], void* gfxContext, bool validateOnly) { @@ -1399,6 +1422,15 @@ void Device::memFree(void* ptr, size_t size) const { } } +void Device::updateFreeMemory(size_t size, bool free) { + if (free) { + freeMem_ += size; + } + else { + freeMem_ -= size; + } +} + void* Device::svmAlloc(amd::Context& context, size_t size, size_t alignment, cl_svm_mem_flags flags, void* svmPtr) const { amd::Memory* mem = nullptr; diff --git a/projects/clr/rocclr/runtime/device/rocm/rocdevice.hpp b/projects/clr/rocclr/runtime/device/rocm/rocdevice.hpp index f148efa76b..7630b70b72 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rocdevice.hpp +++ b/projects/clr/rocclr/runtime/device/rocm/rocdevice.hpp @@ -400,6 +400,9 @@ class Device : public NullDevice { // Lock protect P2P staging operations const std::vector& P2PStages() const { return p2p_stages_; } + // Update the global free memory size + void updateFreeMemory(size_t size, bool free); + private: static hsa_ven_amd_loader_1_00_pfn_t amd_loader_ext_table; @@ -433,6 +436,7 @@ class Device : public NullDevice { XferBuffers* xferWrite_; //!< Transfer buffers write const IProDevice* pro_device_; //!< AMDGPUPro device bool pro_ena_; //!< Extra functionality with AMDGPUPro device, beyond ROCr + std::atomic freeMem_; //!< Total of free memory available public: amd::Atomic numOfVgpus_; //!< Virtual gpu unique index diff --git a/projects/clr/rocclr/runtime/device/rocm/rocmemory.cpp b/projects/clr/rocclr/runtime/device/rocm/rocmemory.cpp index 535880ef5f..ef331176a7 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rocmemory.cpp +++ b/projects/clr/rocclr/runtime/device/rocm/rocmemory.cpp @@ -586,6 +586,7 @@ void Buffer::destroy() { } } else { dev().memFree(deviceMemory_, size()); + const_cast(dev()).updateFreeMemory(size(), true); } } @@ -669,6 +670,9 @@ bool Buffer::create() { deviceMemory_ = dev().hostAlloc(size(), 1, false); owner()->setHostMem(deviceMemory_); } + else { + const_cast(dev()).updateFreeMemory(size(), false); + } assert(amd::isMultipleOf(deviceMemory_, static_cast(dev().info().memBaseAddrAlign_))); @@ -932,6 +936,9 @@ bool Image::create() { if (originalDeviceMemory_ == nullptr) { originalDeviceMemory_ = dev().hostAlloc(alloc_size, 1, false); } + else { + const_cast(dev()).updateFreeMemory(alloc_size, false); + } deviceMemory_ = reinterpret_cast( amd::alignUp(reinterpret_cast(originalDeviceMemory_), deviceImageInfo_.alignment)); @@ -1092,6 +1099,7 @@ void Image::destroy() { if (originalDeviceMemory_ != nullptr) { dev().memFree(originalDeviceMemory_, deviceImageInfo_.size); + const_cast(dev()).updateFreeMemory(size(), true); } } } diff --git a/projects/clr/rocclr/runtime/device/rocm/rocsettings.cpp b/projects/clr/rocclr/runtime/device/rocm/rocsettings.cpp index 94e1e4f666..dfab1f256f 100644 --- a/projects/clr/rocclr/runtime/device/rocm/rocsettings.cpp +++ b/projects/clr/rocclr/runtime/device/rocm/rocsettings.cpp @@ -101,6 +101,8 @@ bool Settings::create(bool fullProfile, int gfxipVersion) { if (MesaInterop::Supported()) { enableExtension(ClKhrGlSharing); } + // Enable platform extension + enableExtension(ClAmdDeviceAttributeQuery); // Enable KHR double precision extension enableExtension(ClKhrFp64);