SWDEV-305894 - Cooperative groups grid and multi grid sync support for gfx940+

Change-Id: I35d72f1cb50c3a96eee56a612b72d641852b145f [ROCm/clr commit: 5d7f3f9f3c]
2022-11-30 19:59:34 +00:00
commit 7aa958a8f7
@@ -1533,6 +1533,8 @@ class Device : public RuntimeObject {
    uint32_t num_grids;
    uint64_t prev_sum;
    uint64_t all_sum;
+    struct MGSyncData sgs;
+    uint num_wg;
  };

  //Attributes that could be retrived from hsa_amd_memory_pool_link_info_t.
@@ -1561,6 +1563,7 @@ class Device : public RuntimeObject {
  static constexpr size_t kP2PStagingSize = 4 * Mi;
  static constexpr size_t kMGSyncDataSize = sizeof(MGSyncData);
  static constexpr size_t kMGInfoSizePerDevice = kMGSyncDataSize + sizeof(MGSyncInfo);
+  static constexpr size_t kSGInfoSize = kMGSyncDataSize;

  typedef std::list<CommandQueue*> CommandQueues;

@@ -100,6 +100,8 @@ Settings::Settings() {

  // Use coarse grain system memory for kernel arguments by default (to keep GPU cache)
  fgs_kernel_arg_ = false;
+  // by default for asics < gfx940 old single grid sync path is followed
+  coop_sync_ = false;
 }

 // ================================================================================================
@@ -173,6 +175,10 @@ bool Settings::create(bool fullProfile, uint32_t gfxipMajor, uint32_t gfxipMinor
    enableWave32Mode_ = GPU_ENABLE_WAVE32_MODE;
  }

+  if (gfxipMajor >= 9 && gfxipMinor >= 4 && gfxStepping >= 0) {
+    coop_sync_ = true;
+  }
+
  lcWavefrontSize64_ = !enableWave32Mode_;

  // Override current device settings
@@ -53,7 +53,8 @@ class Settings : public device::Settings {
      uint system_scope_signal_ : 1;    //!< HSA signal is visibile to the entire system
      uint skip_copy_sync_ : 1;         //!< Ignore explicit HSA signal waits for copy functionality
      uint fgs_kernel_arg_ : 1;         //!< Use fine grain kernel arg segment
-      uint reserved_ : 20;
+      uint coop_sync_ : 1;              //!< grid and multi-grid sync for gfx940+
+      uint reserved_ : 19;
    };
    uint value_;
  };
@@ -2901,21 +2901,30 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes,
        case amd::KernelParameterDescriptor::HiddenMultiGridSync: {
          uint64_t gridSync = coopGroups ? 1 : 0;
          bool multiGrid = (vcmd != nullptr) ? vcmd->cooperativeMultiDeviceGroups() : false;
+          Device::MGSyncInfo* syncInfo = nullptr;
          if (multiGrid) {
            // Find CPU pointer to the right sync info structure. It should be after MGSyncData
-            Device::MGSyncInfo* syncInfo = reinterpret_cast<Device::MGSyncInfo*>(
-                dev().MGSync() + Device::kMGInfoSizePerDevice * dev().index() + Device::kMGSyncDataSize);
+            syncInfo = reinterpret_cast<Device::MGSyncInfo*>(
+              dev().MGSync() + Device::kMGInfoSizePerDevice * dev().index() + Device::kMGSyncDataSize);
            // Update sync data address. Use the offset adjustment to the right location
            syncInfo->mgs = reinterpret_cast<Device::MGSyncData*>(dev().MGSync() +
-              Device::kMGInfoSizePerDevice * vcmd->firstDevice());
-            // Fill all sync info fields
-            syncInfo->grid_id = vcmd->gridId();
-            syncInfo->num_grids = vcmd->numGrids();
-            syncInfo->prev_sum = vcmd->prevGridSum();
-            syncInfo->all_sum = vcmd->allGridSum();
-            // Update GPU address for grid sync info. Use the offset adjustment for the right location
-            gridSync = reinterpret_cast<uint64_t>(syncInfo);
+                            Device::kMGInfoSizePerDevice * vcmd->firstDevice());
          }
+          else {
+            syncInfo = reinterpret_cast<Device::MGSyncInfo*>(allocKernArg(Device::kSGInfoSize, 64));
+            syncInfo->mgs = nullptr;
+          }
+          // Update sync data address.
+          syncInfo->sgs = {0};
+          // Fill all sync info fields
+          syncInfo->grid_id = vcmd->gridId();
+          syncInfo->num_grids = vcmd->numGrids();
+          syncInfo->prev_sum = vcmd->prevGridSum();
+          syncInfo->all_sum = vcmd->allGridSum();
+          syncInfo->num_wg = vcmd->numWorkgroups();
+          // Update GPU address for grid sync info. Use the offset adjustment for the right
+          // location
+          gridSync = reinterpret_cast<uint64_t>(syncInfo);
          WriteAqlArgAt(hidden_arguments, gridSync, it.size_, it.offset_);
          break;
        }
@@ -3128,7 +3137,7 @@ bool VirtualGPU::submitKernelInternal(const amd::NDRangeContainer& sizes,
 */
 // ================================================================================================
 void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) {
-  if (vcmd.cooperativeGroups() || vcmd.cooperativeMultiDeviceGroups()) {
+  if (vcmd.cooperativeGroups()) {
    // Wait for the execution on the current queue, since the coop groups will use the device queue
    releaseGpuMemoryFence(kSkipCpuWait);

@@ -3148,15 +3157,8 @@ void VirtualGPU::submitKernel(amd::NDRangeKernelCommand& vcmd) {
    // Add a dependency into the device queue on the current queue
    queue->Barriers().AddExternalSignal(Barriers().GetLastSignal());

-    if (vcmd.cooperativeGroups()) {
-      // Initialize GWS if it's cooperative groups launch
-      uint32_t workgroups = 1;
-      for (uint i = 0; i < vcmd.sizes().dimensions(); i++) {
-        if (vcmd.sizes().local()[i] != 0) {
-          workgroups *= (vcmd.sizes().global()[i] / vcmd.sizes().local()[i]);
-        }
-      }
-
+    if (!Settings().coop_sync_) {
+      uint32_t workgroups = vcmd.numWorkgroups();
      static_cast<KernelBlitManager&>(queue->blitMgr()).RunGwsInit(workgroups - 1);
    }

@@ -428,6 +428,9 @@ NDRangeKernelCommand::NDRangeKernelCommand(HostQueue& queue, const EventWaitList
  auto devKernel = const_cast<device::Kernel*>(kernel.getDeviceKernel(device));
  profilingInfo_.setCallback(devKernel->getProfilingCallback(
    queue.vdev()), devKernel->getWavesPerSH(queue.vdev()));
+  if (cooperativeGroups()) {
+    setNumWorkgroups();
+  }
  if (forceProfiling) {
    profilingInfo_.enabled_ = true;
    profilingInfo_.clear();
@@ -1050,7 +1050,8 @@ class NDRangeKernelCommand : public Command {
  uint32_t numGrids_;       //!< Total number of grids in multi GPU launch
  uint64_t prevGridSum_;    //!< A sum of previous grids to the current launch
  uint64_t allGridSum_;     //!< A sum of all grids in multi GPU launch
-  uint32_t firstDevice_;    //!< Device index of the first device in the grid
+  uint32_t firstDevice_;    //!< Device index of the first device in the gridc
+  uint32_t numWorkgroups_;  //!< Total number of workgroups in the current launch

 public:
  enum {
@@ -1118,9 +1119,22 @@ class NDRangeKernelCommand : public Command {
  //! Return the index of the first device in multi GPU launch
  uint64_t firstDevice() const { return firstDevice_; }

+  uint32_t numWorkgroups() const { return numWorkgroups_; }
+
  //! Set the local work size.
  void setLocalWorkSize(const NDRange& local) { sizes_.local() = local; }

+  //! Set the number of workgroups
+  void setNumWorkgroups() {
+    uint32_t numWorkgroups = 1;
+    for (uint i = 0; i < sizes().dimensions(); i++) {
+      if (sizes().local()[i] != 0) {
+        numWorkgroups *= (sizes().global()[i] / sizes().local()[i]);
+      }
+    }
+    numWorkgroups_ = numWorkgroups;
+  }
+
  int32_t captureAndValidate();
 };