SWDEV-305894 - Cooperative groups grid and multi grid sync support for gfx940+

Change-Id: I35d72f1cb50c3a96eee56a612b72d641852b145f


[ROCm/clr commit: 5d7f3f9f3c]

projects/clr/rocclr/device/device.hpp

@@ -1533,6 +1533,8 @@ class Device : public RuntimeObject {
     uint32_t num_grids;
     uint64_t prev_sum;
     uint64_t all_sum;
+    struct MGSyncData sgs;
+    uint num_wg;
   };
 
   //Attributes that could be retrived from hsa_amd_memory_pool_link_info_t.
@@ -1561,6 +1563,7 @@ class Device : public RuntimeObject {
   static constexpr size_t kP2PStagingSize = 4 * Mi;
   static constexpr size_t kMGSyncDataSize = sizeof(MGSyncData);
   static constexpr size_t kMGInfoSizePerDevice = kMGSyncDataSize + sizeof(MGSyncInfo);
+  static constexpr size_t kSGInfoSize = kMGSyncDataSize;
 
   typedef std::list<CommandQueue*> CommandQueues;
 

projects/clr/rocclr/device/rocm/rocsettings.cpp

@@ -100,6 +100,8 @@ Settings::Settings() {
 
   // Use coarse grain system memory for kernel arguments by default (to keep GPU cache)
   fgs_kernel_arg_ = false;
+  // by default for asics < gfx940 old single grid sync path is followed
+  coop_sync_ = false;
 }
 
 // ================================================================================================
@@ -173,6 +175,10 @@ bool Settings::create(bool fullProfile, uint32_t gfxipMajor, uint32_t gfxipMinor
     enableWave32Mode_ = GPU_ENABLE_WAVE32_MODE;
   }
 
+  if (gfxipMajor >= 9 && gfxipMinor >= 4 && gfxStepping >= 0) {
+    coop_sync_ = true;
+  }
+
   lcWavefrontSize64_ = !enableWave32Mode_;
 
   // Override current device settings

projects/clr/rocclr/device/rocm/rocsettings.hpp

@@ -53,7 +53,8 @@ class Settings : public device::Settings {
       uint system_scope_signal_ : 1;    //!< HSA signal is visibile to the entire system
       uint skip_copy_sync_ : 1;         //!< Ignore explicit HSA signal waits for copy functionality
       uint fgs_kernel_arg_ : 1;         //!< Use fine grain kernel arg segment
-      uint reserved_ : 20;
+      uint coop_sync_ : 1;              //!< grid and multi-grid sync for gfx940+
+      uint reserved_ : 19;
     };
     uint value_;
   };

projects/clr/rocclr/device/rocm/rocvirtual.cpp

@@ -2901,21 +2901,30 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes,
         case amd::KernelParameterDescriptor::HiddenMultiGridSync: {
           uint64_t gridSync = coopGroups ? 1 : 0;
           bool multiGrid = (vcmd != nullptr) ? vcmd->cooperativeMultiDeviceGroups() : false;
+          Device::MGSyncInfo* syncInfo = nullptr;
           if (multiGrid) {
             // Find CPU pointer to the right sync info structure. It should be after MGSyncData
-            Device::MGSyncInfo* syncInfo = reinterpret_cast<Device::MGSyncInfo*>(
-                dev().MGSync() + Device::kMGInfoSizePerDevice * dev().index() + Device::kMGSyncDataSize);
+            syncInfo = reinterpret_cast<Device::MGSyncInfo*>(
+              dev().MGSync() + Device::kMGInfoSizePerDevice * dev().index() + Device::kMGSyncDataSize);
             // Update sync data address. Use the offset adjustment to the right location
             syncInfo->mgs = reinterpret_cast<Device::MGSyncData*>(dev().MGSync() +
-              Device::kMGInfoSizePerDevice * vcmd->firstDevice());
-            // Fill all sync info fields
-            syncInfo->grid_id = vcmd->gridId();
-            syncInfo->num_grids = vcmd->numGrids();
-            syncInfo->prev_sum = vcmd->prevGridSum();
-            syncInfo->all_sum = vcmd->allGridSum();
-            // Update GPU address for grid sync info. Use the offset adjustment for the right location
-            gridSync = reinterpret_cast<uint64_t>(syncInfo);
+                            Device::kMGInfoSizePerDevice * vcmd->firstDevice());
           }
+          else {
+            syncInfo = reinterpret_cast<Device::MGSyncInfo*>(allocKernArg(Device::kSGInfoSize, 64));
+            syncInfo->mgs = nullptr;
+          }
+          // Update sync data address.
+          syncInfo->sgs = {0};
+          // Fill all sync info fields
+          syncInfo->grid_id = vcmd->gridId();
+          syncInfo->num_grids = vcmd->numGrids();
+          syncInfo->prev_sum = vcmd->prevGridSum();
+          syncInfo->all_sum = vcmd->allGridSum();
+          syncInfo->num_wg = vcmd->numWorkgroups();
+          // Update GPU address for grid sync info. Use the offset adjustment for the right
+          // location
+          gridSync = reinterpret_cast<uint64_t>(syncInfo);
           WriteAqlArgAt(hidden_arguments, gridSync, it.size_, it.offset_);
           break;
         }
@@ -3128,7 +3137,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes,
  */
  // ================================================================================================
 void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) {
-  if (vcmd.cooperativeGroups() || vcmd.cooperativeMultiDeviceGroups()) {
+  if (vcmd.cooperativeGroups()) {
     // Wait for the execution on the current queue, since the coop groups will use the device queue
     releaseGpuMemoryFence(kSkipCpuWait);
 
@@ -3148,15 +3157,8 @@ void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) {
     // Add a dependency into the device queue on the current queue
     queue->Barriers().AddExternalSignal(Barriers().GetLastSignal());
 
-    if (vcmd.cooperativeGroups()) {
-      // Initialize GWS if it's cooperative groups launch
-      uint32_t workgroups = 1;
-      for (uint i = 0; i < vcmd.sizes().dimensions(); i++) {
-        if (vcmd.sizes().local()[i] != 0) {
-          workgroups *= (vcmd.sizes().global()[i] / vcmd.sizes().local()[i]);
-        }
-      }
-
+    if (!Settings().coop_sync_) {
+      uint32_t workgroups = vcmd.numWorkgroups();
       static_cast<KernelBlitManager&>(queue->blitMgr()).RunGwsInit(workgroups - 1);
     }