diff --git a/projects/clr/rocclr/runtime/device/gpu/gpublit.cpp b/projects/clr/rocclr/runtime/device/gpu/gpublit.cpp index c8f5eab22c..d5116992de 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpublit.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpublit.cpp @@ -2727,7 +2727,7 @@ KernelBlitManager::runScheduler( device::Memory& vqueue, device::Memory& params, uint paramIdx, - uint numSlots + uint threads ) const { amd::ScopedLock k(lockXferOps_); @@ -2735,7 +2735,7 @@ KernelBlitManager::runScheduler( size_t dim = 1; size_t globalWorkOffset[1] = { 0 }; - size_t globalWorkSize[1] = { numSlots / 32 }; + size_t globalWorkSize[1] = { threads }; size_t localWorkSize[1] = { 1 }; // Program kernels arguments diff --git a/projects/clr/rocclr/runtime/device/gpu/gpublit.hpp b/projects/clr/rocclr/runtime/device/gpu/gpublit.hpp index 5b3fb86d13..5abfe0b5ad 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpublit.hpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpublit.hpp @@ -379,7 +379,7 @@ public: device::Memory& vqueue, //!< Memory object for virtual queue device::Memory& params, //!< Extra arguments for the scheduler uint paramIdx, //!< Parameter index - uint numSlots //!< Number of slots in the queue + uint threads //!< Number of scheduling threads ) const; private: diff --git a/projects/clr/rocclr/runtime/device/gpu/gpudefs.hpp b/projects/clr/rocclr/runtime/device/gpu/gpudefs.hpp index d919caf182..ad8a73c5c1 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpudefs.hpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpudefs.hpp @@ -61,6 +61,9 @@ const static uint HsaImageObjectAlignment = 16; const static uint HsaSamplerObjectSize = 32; const static uint HsaSamplerObjectAlignment = 16; +//! HSA path specific defines for images +const static uint DeviceQueueMaskSize = 32; + //! Defines all supported ASIC families enum AsicFamilies { Family7xx, diff --git a/projects/clr/rocclr/runtime/device/gpu/gpudevice.cpp b/projects/clr/rocclr/runtime/device/gpu/gpudevice.cpp index d1dd95bdf8..d88b3227bd 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpudevice.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpudevice.cpp @@ -490,7 +490,7 @@ void NullDevice::fillDeviceInfo( info_.queueOnDeviceProperties_ = CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE | CL_QUEUE_PROFILING_ENABLE; info_.queueOnDevicePreferredSize_ = 256 * Ki; - info_.queueOnDeviceMaxSize_ = 512 * Ki; + info_.queueOnDeviceMaxSize_ = 12 * Mi; info_.maxOnDeviceQueues_ = 1; info_.maxOnDeviceEvents_ = settings().numDeviceEvents_; info_.globalVariablePreferredTotalSize_ = static_cast(info_.globalMemSize_); diff --git a/projects/clr/rocclr/runtime/device/gpu/gpusched.hpp b/projects/clr/rocclr/runtime/device/gpu/gpusched.hpp index 7304351f6b..8776957726 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpusched.hpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpusched.hpp @@ -70,6 +70,8 @@ struct SchedulerParam { uint64_t parentAQL; //!< Host parent AmdAqlWrap packet uint32_t dedicatedQueue; //!< Scheduler uses a dedicated queue uint32_t scratchOffset; //!< Scratch buffer offset + uint32_t mask_groups; //!< Processed mask groups by one thread + uint32_t reserved; //!< Reserved }; } // namespace gpu diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp b/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp index 60c9e9575d..74dee7aa67 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.cpp @@ -278,9 +278,18 @@ VirtualGPU::createVirtualQueue(uint deviceQueueSize) { uint MinDeviceQueueSize = 16 * 1024; deviceQueueSize = std::max(deviceQueueSize, MinDeviceQueueSize); + + maskGroups_ = deviceQueueSize / (512 * Ki); + maskGroups_ = (maskGroups_== 0) ? 1 : maskGroups_; + // Align the queue size for the multiple dispatch scheduler. - // Each thread works with 32 entries - deviceQueueSize = amd::alignUp(deviceQueueSize, sizeof(AmdAqlWrap) * 32); + // Each thread works with 32 entries * maskGroups + uint extra = deviceQueueSize % (sizeof(AmdAqlWrap) * + DeviceQueueMaskSize * maskGroups_); + if (extra != 0) { + deviceQueueSize += (sizeof(AmdAqlWrap) * + DeviceQueueMaskSize * maskGroups_) - extra; + } if (deviceQueueSize_ == deviceQueueSize) { return true; @@ -319,11 +328,11 @@ VirtualGPU::createVirtualQueue(uint deviceQueueSize) uint eventMaskOffs = allocSize; // Add mask array for events - allocSize += amd::alignUp(dev().settings().numDeviceEvents_, 32) / 8; + allocSize += amd::alignUp(dev().settings().numDeviceEvents_, DeviceQueueMaskSize) / 8; uint slotMaskOffs = allocSize; // Add mask array for AmdAqlWrap slots - allocSize += amd::alignUp(numSlots, 32) / 8; + allocSize += amd::alignUp(numSlots, DeviceQueueMaskSize) / 8; virtualQueue_ = new Memory(dev(), allocSize); Resource::MemoryType type = (GPU_PRINT_CHILD_KERNEL == 0) ? @@ -402,6 +411,7 @@ VirtualGPU::VirtualGPU( , schedParams_(NULL) , schedParamIdx_(0) , deviceQueueSize_(0) + , maskGroups_(1) , hsaQueueMem_(NULL) , profileEnabled_(false) { @@ -1908,7 +1918,7 @@ VirtualGPU::submitKernelInternalHSA( static_cast(gpuDefQueue->blitMgr()).runScheduler( *gpuDefQueue->virtualQueue_, *gpuDefQueue->schedParams_, gpuDefQueue->schedParamIdx_, - gpuDefQueue->vqHeader_->aql_slot_num); + gpuDefQueue->vqHeader_->aql_slot_num / (DeviceQueueMaskSize * maskGroups_)); const static bool FlushL2 = true; gpuDefQueue->flushCUCaches(FlushL2); @@ -1928,6 +1938,7 @@ VirtualGPU::submitKernelInternalHSA( param->parentAQL = vmParentWrap; param->dedicatedQueue = dev().settings().useDeviceQueue_; param->useATC = dev().settings().svmFineGrainSystem_; + param->mask_groups = maskGroups_; // Fill the scratch buffer information if (hsaKernel.prog().maxScratchRegs() > 0) { @@ -1958,7 +1969,8 @@ VirtualGPU::submitKernelInternalHSA( gpuDefQueue->schedParamIdx_ * sizeof(SchedulerParam); gpuDefQueue->virtualQueueDispatcherEnd(gpuEvent, gpuDefQueue->vmMems(), gpuDefQueue->cal_.memCount_, - signalAddr, loopStart, gpuDefQueue->vqHeader_->aql_slot_num / 32); + signalAddr, loopStart, gpuDefQueue->vqHeader_->aql_slot_num / + (DeviceQueueMaskSize * maskGroups_)); // Set GPU event for the used resources for (uint i = 0; i < memList.size(); ++i) { diff --git a/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.hpp b/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.hpp index ca2fb034bf..07b2fefa04 100644 --- a/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.hpp +++ b/projects/clr/rocclr/runtime/device/gpu/gpuvirtual.hpp @@ -545,6 +545,7 @@ private: Memory* schedParams_; //!< The scheduler parameters uint schedParamIdx_; //!< Index in the scheduler parameters buffer uint deviceQueueSize_; //!< Device queue size + uint maskGroups_; //!< The number of mask groups processed in the scheduler by one thread Memory* hsaQueueMem_; //!< Memory for the amd_queue_t object bool profileEnabled_;//!< Profiling is enabled