From 896a035951e26eaef56893c95a290b4fd2190b57 Mon Sep 17 00:00:00 2001
From: Sean Keely <Sean.Keely@amd.com>
Date: Wed, 8 Aug 2018 20:15:22 -0500
Subject: [PATCH] Experimental flag to swap copy agent for async copy APIs.

Adds env flag HSA_REV_COPY_DIR.  If set to 1 async copy will
copy from dst device to src device rather than from src to dst.

Change-Id: I3095642066fa026dc112c2eac06db9393341cd7e


[ROCm/ROCR-Runtime commit: 6c4778062048dda981919e198c5f22ab2fd05121]
---
 .../runtime/hsa-runtime/core/runtime/runtime.cpp          | 8 +++++---
 .../rocr-runtime/runtime/hsa-runtime/core/util/flag.h     | 6 ++++++
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp
index af94a1ae64..d2ed99119c 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp
@@ -416,9 +416,11 @@ hsa_status_t Runtime::CopyMemory(void* dst, core::Agent& dst_agent,
   const bool src_gpu =
       (src_agent.device_type() == core::Agent::DeviceType::kAmdGpuDevice);
   if (dst_gpu || src_gpu) {
-    core::Agent& copy_agent = (src_gpu) ? src_agent : dst_agent;
-    return copy_agent.DmaCopy(dst, dst_agent, src, src_agent, size, dep_signals,
-                              completion_signal);
+    core::Agent* copy_agent = (src_gpu) ? &src_agent : &dst_agent;
+    if (flag_.rev_copy_dir() && dst_gpu && src_gpu)
+      copy_agent = (copy_agent == &src_agent) ? &dst_agent : &src_agent;
+    return copy_agent->DmaCopy(dst, dst_agent, src, src_agent, size, dep_signals,
+                               completion_signal);
   }
 
   // For cpu to cpu, fire and forget a copy thread.
diff --git a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h
index 6dd81ac9b5..8d0e1af560 100644
--- a/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h
+++ b/projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h
@@ -97,6 +97,9 @@ class Flag {
 
     var = os::GetEnvVar("HSA_ENABLE_SDMA_HDP_FLUSH");
     enable_sdma_hdp_flush_ = (var == "0") ? false : true;
+
+    var = os::GetEnvVar("HSA_REV_COPY_DIR");
+    rev_copy_dir_ = (var == "1") ? true : false;
   }
 
   bool check_flat_scratch() const { return check_flat_scratch_; }
@@ -117,6 +120,8 @@ class Flag {
 
   bool disable_fragment_alloc() const { return disable_fragment_alloc_; }
 
+  bool rev_copy_dir() const { return rev_copy_dir_; }
+
   std::string enable_sdma() const { return enable_sdma_; }
 
   uint32_t max_queues() const { return max_queues_; }
@@ -135,6 +140,7 @@ class Flag {
   bool enable_queue_fault_message_;
   bool report_tool_load_failures_;
   bool disable_fragment_alloc_;
+  bool rev_copy_dir_;
 
   std::string enable_sdma_;