Experimental flag to swap copy agent for async copy APIs.

Adds env flag HSA_REV_COPY_DIR.  If set to 1 async copy will
copy from dst device to src device rather than from src to dst.

Change-Id: I3095642066fa026dc112c2eac06db9393341cd7e


[ROCm/ROCR-Runtime commit: 6c47780620]

projects/rocr-runtime/runtime/hsa-runtime/core/runtime/runtime.cpp

@@ -416,9 +416,11 @@ hsa_status_t Runtime::CopyMemory(void* dst, core::Agent& dst_agent,
   const bool src_gpu =
       (src_agent.device_type() == core::Agent::DeviceType::kAmdGpuDevice);
   if (dst_gpu || src_gpu) {
-    core::Agent& copy_agent = (src_gpu) ? src_agent : dst_agent;
-    return copy_agent.DmaCopy(dst, dst_agent, src, src_agent, size, dep_signals,
-                              completion_signal);
+    core::Agent* copy_agent = (src_gpu) ? &src_agent : &dst_agent;
+    if (flag_.rev_copy_dir() && dst_gpu && src_gpu)
+      copy_agent = (copy_agent == &src_agent) ? &dst_agent : &src_agent;
+    return copy_agent->DmaCopy(dst, dst_agent, src, src_agent, size, dep_signals,
+                               completion_signal);
   }
 
   // For cpu to cpu, fire and forget a copy thread.

projects/rocr-runtime/runtime/hsa-runtime/core/util/flag.h

@@ -97,6 +97,9 @@ class Flag {
 
     var = os::GetEnvVar("HSA_ENABLE_SDMA_HDP_FLUSH");
     enable_sdma_hdp_flush_ = (var == "0") ? false : true;
+
+    var = os::GetEnvVar("HSA_REV_COPY_DIR");
+    rev_copy_dir_ = (var == "1") ? true : false;
   }
 
   bool check_flat_scratch() const { return check_flat_scratch_; }
@@ -117,6 +120,8 @@ class Flag {
 
   bool disable_fragment_alloc() const { return disable_fragment_alloc_; }
 
+  bool rev_copy_dir() const { return rev_copy_dir_; }
+
   std::string enable_sdma() const { return enable_sdma_; }
 
   uint32_t max_queues() const { return max_queues_; }
@@ -135,6 +140,7 @@ class Flag {
   bool enable_queue_fault_message_;
   bool report_tool_load_failures_;
   bool disable_fragment_alloc_;
+  bool rev_copy_dir_;
 
   std::string enable_sdma_;