From fe47915a8e61be55caa8e05083d2a37441857163 Mon Sep 17 00:00:00 2001
From: Sean Keely <Sean.Keely@amd.com>
Date: Thu, 3 Sep 2020 03:06:03 -0500
Subject: [PATCH] Use SDMA for small copies in VRAM.

For small copies cache flush latency is larger than data transfer
latency in local VRAM.  Select SDMA for small copies.

Environment key HSA_FORCE_SDMA_SIZE is added for easy adjustment
of the small copy size.  This may be removed after tuning is done.

Change-Id: I733fa0ae01c616617c5de50e71226b51fd589ef2


[ROCm/ROCR-Runtime commit: 2a0c6774fb6e3ab9ed42c5c233e3f015013de685]
---
 .../runtime/hsa-runtime/core/inc/amd_gpu_agent.h         | 3 ++-
 .../runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp   | 9 +++++++--
 .../rocr-runtime/runtime/hsa-runtime/core/util/flag.h    | 7 +++++++
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h
index 91c1119c95..b86f706d75 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/inc/amd_gpu_agent.h
@@ -502,7 +502,8 @@ class GpuAgent : public GpuAgentInt {
   lazy_ptr<core::Blit>& GetPcieBlit(const core::Agent& dst_agent, const core::Agent& src_agent);
 
   // Bind the Blit object that will drive the copy operation
-  lazy_ptr<core::Blit>& GetBlitObject(const core::Agent& dst_agent, const core::Agent& src_agent);
+  lazy_ptr<core::Blit>& GetBlitObject(const core::Agent& dst_agent, const core::Agent& src_agent,
+                                      const size_t size);
   // @brief Alternative aperture base address. Only on KV.
   uintptr_t ape1_base_;
 
diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
index 2caf5fa07d..468d1142d7 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/amd_gpu_agent.cpp
@@ -645,7 +645,7 @@ hsa_status_t GpuAgent::DmaCopy(void* dst, core::Agent& dst_agent,
                                std::vector<core::Signal*>& dep_signals,
                                core::Signal& out_signal) {
   // Bind the Blit object that will drive this copy operation
-  lazy_ptr<core::Blit>& blit = GetBlitObject(dst_agent, src_agent);
+  lazy_ptr<core::Blit>& blit = GetBlitObject(dst_agent, src_agent, size);
 
   if (profiling_enabled()) {
     // Track the agent so we could translate the resulting timestamp to system
@@ -1354,7 +1354,7 @@ lazy_ptr<core::Blit>& GpuAgent::GetPcieBlit(const core::Agent& dst_agent,
 }
 
 lazy_ptr<core::Blit>& GpuAgent::GetBlitObject(const core::Agent& dst_agent,
-                                              const core::Agent& src_agent) {
+                                              const core::Agent& src_agent, const size_t size) {
   // At this point it is guaranteed that one of
   // the two devices is a GPU, potentially both
   assert(((src_agent.device_type() == core::Agent::kAmdGpuDevice) ||
@@ -1363,6 +1363,11 @@ lazy_ptr<core::Blit>& GpuAgent::GetBlitObject(const core::Agent& dst_agent,
 
   // Determine if Src and Dst devices are same
   if ((src_agent.public_handle().handle) == (dst_agent.public_handle().handle)) {
+    // If the copy is very small then cache flush overheads can dominate.
+    // Choose a (potentially) SDMA enabled engine to avoid cache flushing.
+    if (size < core::Runtime::runtime_singleton_->flag().force_sdma_size()) {
+      return blits_[BlitDevToHost];
+    }
     return blits_[BlitDevToDev];
   }
 
diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h
index f43c6da32a..f3b4141c9d 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h
@@ -121,6 +121,9 @@ class Flag {
 
     var = os::GetEnvVar("HSA_LOADER_ENABLE_MMAP_URI");
     loader_enable_mmap_uri_ = (var == "1") ? true : false;
+
+    var = os::GetEnvVar("HSA_FORCE_SDMA_SIZE");
+    force_sdma_size_ = var.empty() ? 1024 * 1024 : atoi(var.c_str());
   }
 
   bool check_flat_scratch() const { return check_flat_scratch_; }
@@ -165,6 +168,8 @@ class Flag {
 
   bool loader_enable_mmap_uri() const { return loader_enable_mmap_uri_; }
 
+  size_t force_sdma_size() const { return force_sdma_size_; }
+
  private:
   bool check_flat_scratch_;
   bool enable_vm_fault_message_;
@@ -193,6 +198,8 @@ class Flag {
 
   std::string tools_lib_names_;
 
+  size_t force_sdma_size_;
+
   DISALLOW_COPY_AND_ASSIGN(Flag);
 };