Increase SDMA copy size

SDMA4.4 and SDMA5.2+ has increased it's available copy size to 2^30 bytes
represented by exponent as bits set in the COUNT field of the
linear copy.

Also note that the full 2^22 byte limit is available from SDMA4 onwards
as it has corrected the 0x3fffe0 HW limitation from SDMA3.

As copy limit has increase, this can change system performance
so provide env var HSA_ENABLE_SDMA_COPY_SIZE_OVERRIDE=0 to fall
back to the original 0x3fffe0 limit for debugging purposes.

Change-Id: I0fb6e5378f68e5b8a00ff559271691a943ee06ee


[ROCm/ROCR-Runtime commit: 81c64228e0]

projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_blit_sdma.h

@@ -65,7 +65,8 @@ class BlitSdmaBase : public core::Blit {
   static const size_t kMaxSingleCopySize;
   static const size_t kMaxSingleFillSize;
   virtual bool isSDMA() const override { return true; }
-  virtual hsa_status_t Initialize(const core::Agent& agent, bool use_xgmi) = 0;
+  virtual hsa_status_t Initialize(const core::Agent& agent, bool use_xgmi,
+                                  size_t linear_copy_size_override) = 0;
   virtual hsa_status_t SubmitCopyRectCommand(const hsa_pitched_ptr_t* dst,
                                              const hsa_dim3_t* dst_offset,
                                              const hsa_pitched_ptr_t* src,
@@ -90,7 +91,8 @@ class BlitSdma : public BlitSdmaBase {
   /// @param agent Pointer to the agent that will execute the PM4 commands.
   ///
   /// @return hsa_status_t
-  virtual hsa_status_t Initialize(const core::Agent& agent, bool use_xgmi) override;
+  virtual hsa_status_t Initialize(const core::Agent& agent, bool use_xgmi,
+                                  size_t linear_copy_size_override) override;
 
   /// @brief Marks the queue object as invalid and uncouples its link with
   /// the underlying compute device's control block. Use of queue object

projects/rocr-runtime/runtime/hsa-runtime/core/inc/sdma_registers.h

@@ -80,7 +80,11 @@ typedef struct SDMA_PKT_COPY_LINEAR_TAG {
     struct {
       unsigned int count : 22;
       unsigned int reserved_0 : 10;
-    };
+    } count;
+    struct {
+      unsigned int count : 30;
+      unsigned int reserved_0 : 2;
+    } count_ext;
     unsigned int DW_1_DATA;
   } COUNT_UNION;
 

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_blit_sdma.cpp

@@ -131,7 +131,7 @@ BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset, useGCR>::~BlitSdma()
 
 template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset, bool useGCR>
 hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset, useGCR>::Initialize(
-    const core::Agent& agent, bool use_xgmi) {
+    const core::Agent& agent, bool use_xgmi, size_t linear_copy_size_override) {
   if (queue_start_addr_ != NULL) {
     // Already initialized.
     return HSA_STATUS_SUCCESS;
@@ -201,6 +201,8 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset, useGCR>:
   signals_[0].reset(new core::InterruptSignal(0));
   signals_[1].reset(new core::InterruptSignal(0));
 
+  max_single_linear_copy_size_ = linear_copy_size_override;
+
   cleanupOnException.Dismiss();
   return HSA_STATUS_SUCCESS;
 }
@@ -492,7 +494,9 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset,
                       useGCR>::SubmitLinearCopyCommand(void* dst, const void* src, size_t size) {
   // Break the copy into multiple copy operation incase the copy size exceeds
   // the SDMA linear copy limit.
-  const uint32_t num_copy_command = (size + kMaxSingleCopySize - 1) / kMaxSingleCopySize;
+  const size_t max_copy_size = max_single_linear_copy_size_ ? max_single_linear_copy_size_ :
+                               kMaxSingleCopySize;
+  const uint32_t num_copy_command = (size + max_copy_size - 1) / max_copy_size;
 
   std::vector<SDMA_PKT_COPY_LINEAR> buff(num_copy_command);
   BuildCopyCommand(reinterpret_cast<char*>(&buff[0]), num_copy_command, dst, src, size);
@@ -508,7 +512,9 @@ hsa_status_t BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset,
                                                        std::vector<core::Signal*>& gang_signals) {
   // Break the copy into multiple copy operations when the copy size exceeds
   // the SDMA linear copy limit.
-  const uint32_t num_copy_command = (size + kMaxSingleCopySize - 1) / kMaxSingleCopySize;
+  const size_t max_copy_size = max_single_linear_copy_size_ ? max_single_linear_copy_size_ :
+                               kMaxSingleCopySize;
+  const uint32_t num_copy_command = (size + max_copy_size - 1) / max_copy_size;
 
   // Assemble copy packets.
   std::vector<SDMA_PKT_COPY_LINEAR> buff(num_copy_command);
@@ -774,9 +780,11 @@ template <typename RingIndexTy, bool HwIndexMonotonic, int SizeToCountOffset, bo
 void BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset, useGCR>::BuildCopyCommand(
     char* cmd_addr, uint32_t num_copy_command, void* dst, const void* src, size_t size) {
   size_t cur_size = 0;
+  const size_t max_copy_size = max_single_linear_copy_size_ ? max_single_linear_copy_size_ :
+                                                              kMaxSingleCopySize;
   for (uint32_t i = 0; i < num_copy_command; ++i) {
     const uint32_t copy_size =
-        static_cast<uint32_t>(std::min((size - cur_size), kMaxSingleCopySize));
+        static_cast<uint32_t>(std::min((size - cur_size), max_copy_size));
 
     void* cur_dst = static_cast<char*>(dst) + cur_size;
     const void* cur_src = static_cast<const char*>(src) + cur_size;
@@ -789,7 +797,10 @@ void BlitSdma<RingIndexTy, HwIndexMonotonic, SizeToCountOffset, useGCR>::BuildCo
     packet_addr->HEADER_UNION.op = SDMA_OP_COPY;
     packet_addr->HEADER_UNION.sub_op = SDMA_SUBOP_COPY_LINEAR;
 
-    packet_addr->COUNT_UNION.count = copy_size + SizeToCountOffset;
+    if (max_copy_size == (1 << 30) -1)
+      packet_addr->COUNT_UNION.count_ext.count = copy_size + SizeToCountOffset;
+    else
+      packet_addr->COUNT_UNION.count.count = copy_size + SizeToCountOffset;
 
     packet_addr->SRC_ADDR_LO_UNION.src_addr_31_0 = ptrlow32(cur_src);
     packet_addr->SRC_ADDR_HI_UNION.src_addr_63_32 = ptrhigh32(cur_src);

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp

@@ -661,6 +661,8 @@ core::Queue* GpuAgent::CreateInterceptibleQueue(void (*callback)(hsa_status_t st
 
 core::Blit* GpuAgent::CreateBlitSdma(bool use_xgmi) {
   AMD::BlitSdmaBase* sdma;
+  size_t copy_size_override = 0;
+  const size_t copy_size_overrides[2] = {0x3fffff, 0x3fffffff};
 
   switch (isa_->GetMajorVersion()) {
     case 7:
@@ -669,19 +671,29 @@ core::Blit* GpuAgent::CreateBlitSdma(bool use_xgmi) {
       break;
     case 9:
       sdma = new BlitSdmaV4();
+      copy_size_override = (isa_->GetMinorVersion() == 0 && isa_->GetStepping() == 10) ||
+                            isa_->GetMinorVersion() > 0 ? copy_size_overrides[1] :
+                                                          copy_size_overrides[0];
       break;
     case 10:
       sdma = new BlitSdmaV5();
+      copy_size_override = isa_->GetMinorVersion() < 3 ? copy_size_overrides[0] :
+                                                         copy_size_overrides[1];
       break;
     case 11:
       sdma = new BlitSdmaV5();
+      copy_size_override = copy_size_overrides[1];
       break;
     default:
       assert(false && "Unexpected device major version.");
       return nullptr;
   }
 
-  if (sdma->Initialize(*this, use_xgmi) != HSA_STATUS_SUCCESS) {
+  Flag::SDMA_OVERRIDE copy_size_override_setting =
+    core::Runtime::runtime_singleton_->flag().enable_sdma_copy_size_override();
+  if (copy_size_override_setting == Flag::SDMA_DISABLE) copy_size_override = 0;
+
+  if (sdma->Initialize(*this, use_xgmi, copy_size_override) != HSA_STATUS_SUCCESS) {
     sdma->Destroy(*this);
     delete sdma;
     sdma = nullptr;

projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h

@@ -94,6 +94,10 @@ class Flag {
     enable_sdma_gang_ = (var == "0") ? SDMA_DISABLE :
                        ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT);
 
+    var = os::GetEnvVar("HSA_ENABLE_SDMA_COPY_SIZE_OVERRIDE");
+    enable_sdma_copy_size_override_ = (var == "0") ? SDMA_DISABLE :
+                                      ((var == "1") ? SDMA_ENABLE : SDMA_DEFAULT);
+
     visible_gpus_ = os::GetEnvVar("ROCR_VISIBLE_DEVICES");
     filter_visible_gpus_ = os::IsEnvVarSet("ROCR_VISIBLE_DEVICES");
 
@@ -236,6 +240,8 @@ class Flag {
 
   SDMA_OVERRIDE enable_sdma_gang() const { return enable_sdma_gang_; }
 
+  SDMA_OVERRIDE enable_sdma_copy_size_override() const { return enable_sdma_copy_size_override_; }
+
   std::string visible_gpus() const { return visible_gpus_; }
 
   bool filter_visible_gpus() const { return filter_visible_gpus_; }
@@ -315,6 +321,7 @@ class Flag {
   SDMA_OVERRIDE enable_sdma_;
   SDMA_OVERRIDE enable_peer_sdma_;
   SDMA_OVERRIDE enable_sdma_gang_;
+  SDMA_OVERRIDE enable_sdma_copy_size_override_;
 
   bool filter_visible_gpus_;
   std::string visible_gpus_;