From 7aa958a8f763462ededb6efc01a65744718c1e4e Mon Sep 17 00:00:00 2001 From: Sourabh Betigeri Date: Wed, 30 Nov 2022 19:59:34 +0000 Subject: [PATCH] SWDEV-305894 - Cooperative groups grid and multi grid sync support for gfx940+ Change-Id: I35d72f1cb50c3a96eee56a612b72d641852b145f [ROCm/clr commit: 5d7f3f9f3cbf6caf8af80a81a2a22d6772d0934b] --- projects/clr/rocclr/device/device.hpp | 3 ++ .../clr/rocclr/device/rocm/rocsettings.cpp | 6 +++ .../clr/rocclr/device/rocm/rocsettings.hpp | 3 +- .../clr/rocclr/device/rocm/rocvirtual.cpp | 42 ++++++++++--------- projects/clr/rocclr/platform/command.cpp | 3 ++ projects/clr/rocclr/platform/command.hpp | 16 ++++++- 6 files changed, 51 insertions(+), 22 deletions(-) diff --git a/projects/clr/rocclr/device/device.hpp b/projects/clr/rocclr/device/device.hpp index ebddc7c5cd..fa3fc81d64 100644 --- a/projects/clr/rocclr/device/device.hpp +++ b/projects/clr/rocclr/device/device.hpp @@ -1533,6 +1533,8 @@ class Device : public RuntimeObject { uint32_t num_grids; uint64_t prev_sum; uint64_t all_sum; + struct MGSyncData sgs; + uint num_wg; }; //Attributes that could be retrived from hsa_amd_memory_pool_link_info_t. @@ -1561,6 +1563,7 @@ class Device : public RuntimeObject { static constexpr size_t kP2PStagingSize = 4 * Mi; static constexpr size_t kMGSyncDataSize = sizeof(MGSyncData); static constexpr size_t kMGInfoSizePerDevice = kMGSyncDataSize + sizeof(MGSyncInfo); + static constexpr size_t kSGInfoSize = kMGSyncDataSize; typedef std::list CommandQueues; diff --git a/projects/clr/rocclr/device/rocm/rocsettings.cpp b/projects/clr/rocclr/device/rocm/rocsettings.cpp index 222198dbec..49a371bbcc 100644 --- a/projects/clr/rocclr/device/rocm/rocsettings.cpp +++ b/projects/clr/rocclr/device/rocm/rocsettings.cpp @@ -100,6 +100,8 @@ Settings::Settings() { // Use coarse grain system memory for kernel arguments by default (to keep GPU cache) fgs_kernel_arg_ = false; + // by default for asics < gfx940 old single grid sync path is followed + coop_sync_ = false; } // ================================================================================================ @@ -173,6 +175,10 @@ bool Settings::create(bool fullProfile, uint32_t gfxipMajor, uint32_t gfxipMinor enableWave32Mode_ = GPU_ENABLE_WAVE32_MODE; } + if (gfxipMajor >= 9 && gfxipMinor >= 4 && gfxStepping >= 0) { + coop_sync_ = true; + } + lcWavefrontSize64_ = !enableWave32Mode_; // Override current device settings diff --git a/projects/clr/rocclr/device/rocm/rocsettings.hpp b/projects/clr/rocclr/device/rocm/rocsettings.hpp index f12e5cd774..df2fe39c8a 100644 --- a/projects/clr/rocclr/device/rocm/rocsettings.hpp +++ b/projects/clr/rocclr/device/rocm/rocsettings.hpp @@ -53,7 +53,8 @@ class Settings : public device::Settings { uint system_scope_signal_ : 1; //!< HSA signal is visibile to the entire system uint skip_copy_sync_ : 1; //!< Ignore explicit HSA signal waits for copy functionality uint fgs_kernel_arg_ : 1; //!< Use fine grain kernel arg segment - uint reserved_ : 20; + uint coop_sync_ : 1; //!< grid and multi-grid sync for gfx940+ + uint reserved_ : 19; }; uint value_; }; diff --git a/projects/clr/rocclr/device/rocm/rocvirtual.cpp b/projects/clr/rocclr/device/rocm/rocvirtual.cpp index 8a3f2e3094..6dd0b7b2a9 100644 --- a/projects/clr/rocclr/device/rocm/rocvirtual.cpp +++ b/projects/clr/rocclr/device/rocm/rocvirtual.cpp @@ -2901,21 +2901,30 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, case amd::KernelParameterDescriptor::HiddenMultiGridSync: { uint64_t gridSync = coopGroups ? 1 : 0; bool multiGrid = (vcmd != nullptr) ? vcmd->cooperativeMultiDeviceGroups() : false; + Device::MGSyncInfo* syncInfo = nullptr; if (multiGrid) { // Find CPU pointer to the right sync info structure. It should be after MGSyncData - Device::MGSyncInfo* syncInfo = reinterpret_cast( - dev().MGSync() + Device::kMGInfoSizePerDevice * dev().index() + Device::kMGSyncDataSize); + syncInfo = reinterpret_cast( + dev().MGSync() + Device::kMGInfoSizePerDevice * dev().index() + Device::kMGSyncDataSize); // Update sync data address. Use the offset adjustment to the right location syncInfo->mgs = reinterpret_cast(dev().MGSync() + - Device::kMGInfoSizePerDevice * vcmd->firstDevice()); - // Fill all sync info fields - syncInfo->grid_id = vcmd->gridId(); - syncInfo->num_grids = vcmd->numGrids(); - syncInfo->prev_sum = vcmd->prevGridSum(); - syncInfo->all_sum = vcmd->allGridSum(); - // Update GPU address for grid sync info. Use the offset adjustment for the right location - gridSync = reinterpret_cast(syncInfo); + Device::kMGInfoSizePerDevice * vcmd->firstDevice()); } + else { + syncInfo = reinterpret_cast(allocKernArg(Device::kSGInfoSize, 64)); + syncInfo->mgs = nullptr; + } + // Update sync data address. + syncInfo->sgs = {0}; + // Fill all sync info fields + syncInfo->grid_id = vcmd->gridId(); + syncInfo->num_grids = vcmd->numGrids(); + syncInfo->prev_sum = vcmd->prevGridSum(); + syncInfo->all_sum = vcmd->allGridSum(); + syncInfo->num_wg = vcmd->numWorkgroups(); + // Update GPU address for grid sync info. Use the offset adjustment for the right + // location + gridSync = reinterpret_cast(syncInfo); WriteAqlArgAt(hidden_arguments, gridSync, it.size_, it.offset_); break; } @@ -3128,7 +3137,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes, */ // ================================================================================================ void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) { - if (vcmd.cooperativeGroups() || vcmd.cooperativeMultiDeviceGroups()) { + if (vcmd.cooperativeGroups()) { // Wait for the execution on the current queue, since the coop groups will use the device queue releaseGpuMemoryFence(kSkipCpuWait); @@ -3148,15 +3157,8 @@ void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) { // Add a dependency into the device queue on the current queue queue->Barriers().AddExternalSignal(Barriers().GetLastSignal()); - if (vcmd.cooperativeGroups()) { - // Initialize GWS if it's cooperative groups launch - uint32_t workgroups = 1; - for (uint i = 0; i < vcmd.sizes().dimensions(); i++) { - if (vcmd.sizes().local()[i] != 0) { - workgroups *= (vcmd.sizes().global()[i] / vcmd.sizes().local()[i]); - } - } - + if (!Settings().coop_sync_) { + uint32_t workgroups = vcmd.numWorkgroups(); static_cast(queue->blitMgr()).RunGwsInit(workgroups - 1); } diff --git a/projects/clr/rocclr/platform/command.cpp b/projects/clr/rocclr/platform/command.cpp index 26336b3180..e7c2d57600 100644 --- a/projects/clr/rocclr/platform/command.cpp +++ b/projects/clr/rocclr/platform/command.cpp @@ -428,6 +428,9 @@ NDRangeKernelCommand::NDRangeKernelCommand(HostQueue& queue, const EventWaitList auto devKernel = const_cast(kernel.getDeviceKernel(device)); profilingInfo_.setCallback(devKernel->getProfilingCallback( queue.vdev()), devKernel->getWavesPerSH(queue.vdev())); + if (cooperativeGroups()) { + setNumWorkgroups(); + } if (forceProfiling) { profilingInfo_.enabled_ = true; profilingInfo_.clear(); diff --git a/projects/clr/rocclr/platform/command.hpp b/projects/clr/rocclr/platform/command.hpp index 6519557351..d223d0e5f2 100644 --- a/projects/clr/rocclr/platform/command.hpp +++ b/projects/clr/rocclr/platform/command.hpp @@ -1050,7 +1050,8 @@ class NDRangeKernelCommand : public Command { uint32_t numGrids_; //!< Total number of grids in multi GPU launch uint64_t prevGridSum_; //!< A sum of previous grids to the current launch uint64_t allGridSum_; //!< A sum of all grids in multi GPU launch - uint32_t firstDevice_; //!< Device index of the first device in the grid + uint32_t firstDevice_; //!< Device index of the first device in the gridc + uint32_t numWorkgroups_; //!< Total number of workgroups in the current launch public: enum { @@ -1118,9 +1119,22 @@ class NDRangeKernelCommand : public Command { //! Return the index of the first device in multi GPU launch uint64_t firstDevice() const { return firstDevice_; } + uint32_t numWorkgroups() const { return numWorkgroups_; } + //! Set the local work size. void setLocalWorkSize(const NDRange& local) { sizes_.local() = local; } + //! Set the number of workgroups + void setNumWorkgroups() { + uint32_t numWorkgroups = 1; + for (uint i = 0; i < sizes().dimensions(); i++) { + if (sizes().local()[i] != 0) { + numWorkgroups *= (sizes().global()[i] / sizes().local()[i]); + } + } + numWorkgroups_ = numWorkgroups; + } + int32_t captureAndValidate(); };