Update Copy requests involving all pools i.e. options -a or -A

Change-Id: I0c8d8fbb39f43cd6a1f84ae6ae32337fa9b1f5e2


[ROCm/ROCR-Runtime commit: 703b1466c1]

projects/rocr-runtime/rocrtst/samples/rocm_async/CMakeLists.txt

@@ -10,6 +10,8 @@ cmake_minimum_required(VERSION 2.8.0)
 #
 #     export ROCR_LIB_DIR="Path to ROC Runtime libraries"
 #
+#     export ROCT_LIB_DIR="Path to ROC Thunk libraries"
+#
 #  2) Make an new folder called build under root folder
 #
 #     mkdir build
@@ -38,6 +40,8 @@ SET( CMAKE_VERBOSE_MAKEFILE on )
 #
 # Set core runtime module name
 #
+set ( ROC_THUNK_NAME "hsakmt" )
+set ( ROC_THUNK_LIBRARY "lib${ROC_THUNK_NAME}" )
 set ( CORE_RUNTIME_NAME "hsa-runtime" )
 set ( CORE_RUNTIME_TARGET "${CORE_RUNTIME_NAME}64" )
 set ( CORE_RUNTIME_LIBRARY "lib${CORE_RUNTIME_TARGET}" )
@@ -47,6 +51,11 @@ if(NOT EXISTS $ENV{ROCR_LIB_DIR}/${CORE_RUNTIME_LIBRARY}.so)
   RETURN()
 endif()
 
+if(NOT EXISTS $ENV{ROCT_LIB_DIR}/${ROC_THUNK_LIBRARY}.so)
+  MESSAGE("ERROR: ROC Thunk libraries can't be found under sprcified path")
+  RETURN()
+endif()
+
 set(PROJECT_NAME "rocm_async")
 set(TEST_NAME "${PROJECT_NAME}")
 project (${PROJECT_NAME})

projects/rocr-runtime/rocrtst/samples/rocm_async/rocm_async.cpp

@@ -385,12 +385,6 @@ void RocmAsync::RunCopyBenchmark(async_trans_t& trans) {
   }
 }
 
-void RocmAsync::RunIOBenchmark(async_trans_t& trans) {
-
-  std::cout << "Unsupported Request - Read / Write" << std::endl;
-  exit(1);
-}
-
 void RocmAsync::Run() {
 
   // Enable profiling of Async Copy Activity
@@ -402,8 +396,11 @@ void RocmAsync::Run() {
   for (uint32_t idx = 0; idx < trans_size; idx++) {
     async_trans_t& trans = trans_list_[idx];
     if ((trans.req_type_ == REQ_COPY_BIDIR) ||
-        (trans.req_type_ == REQ_COPY_UNIDIR)) {
+        (trans.req_type_ == REQ_COPY_UNIDIR) ||
+        (trans.req_type_ == REQ_COPY_ALL_BIDIR) ||
+        (trans.req_type_ == REQ_COPY_ALL_UNIDIR)) {
       RunCopyBenchmark(trans);
+      ComputeCopyTime(trans);
     }
     if ((trans.req_type_ == REQ_READ) ||
         (trans.req_type_ == REQ_WRITE)) {
@@ -451,15 +448,6 @@ void RocmAsync::SetUp() {
     PrintHelpScreen();
     exit(1);
   }
-
-  // Print Debug Info - List of Agents, Pool, Transactions
-  char* print_debug = getenv("PRINT_DEBUG");
-  if (print_debug) {
-    //PrintAgentsList();
-    //PrintPoolsList();
-    PrintTransList();
-    //PrintTopology();
-  }
 }
 
 RocmAsync::RocmAsync(int argc, char** argv) : BaseTest() {
@@ -472,6 +460,8 @@ RocmAsync::RocmAsync(int argc, char** argv) : BaseTest() {
   req_write_ = REQ_INVALID;
   req_copy_bidir_ = REQ_INVALID;
   req_copy_unidir_ = REQ_INVALID;
+  req_copy_all_bidir_ = REQ_INVALID;
+  req_copy_all_unidir_ = REQ_INVALID;
 }
 
 RocmAsync::~RocmAsync() { }

projects/rocr-runtime/rocrtst/samples/rocm_async/rocm_async.hpp

@@ -106,6 +106,14 @@ typedef struct async_trans {
   // Gpu Min time
   vector<double> gpu_min_time_;
 
+  // BenchMark's Average copy time and average bandwidth
+  vector<double> avg_time_;
+  vector<double> avg_bandwidth_;
+
+  // BenchMark's Min copy time and peak bandwidth
+  vector<double> min_time_;
+  vector<double> peak_bandwidth_;
+
   async_trans(uint32_t req_type) { req_type_ = req_type; }
 } async_trans_t;
 
@@ -115,7 +123,9 @@ typedef enum Request_Type {
   REQ_WRITE = 2,
   REQ_COPY_BIDIR = 3,
   REQ_COPY_UNIDIR = 4,
-  REQ_INVALID = 5,
+  REQ_COPY_ALL_BIDIR = 5,
+  REQ_COPY_ALL_UNIDIR = 6,
+  REQ_INVALID = 7,
 
 } Request_Type;
 
@@ -183,6 +193,7 @@ class RocmAsync : public BaseTest {
   // @brief: Dispaly Benchmark result
   void DisplayIOTime(async_trans_t& trans) const;
   void DisplayCopyTime(async_trans_t& trans) const;
+  void DisplayCopyTimeMatrix() const;
 
   private:
 
@@ -202,11 +213,14 @@ class RocmAsync : public BaseTest {
   bool PoolIsDuplicated(vector<uint32_t>& in_list);
 
   // @brief: Builds a list of transaction per user request
+  void ComputeCopyTime(async_trans_t& trans);
   bool BuildTransList();
   bool BuildReadTrans();
   bool BuildWriteTrans();
   bool BuildBidirCopyTrans();
   bool BuildUnidirCopyTrans();
+  bool BuildAllPoolsBidirCopyTrans();
+  bool BuildAllPoolsUnidirCopyTrans();
   bool BuildReadOrWriteTrans(uint32_t req_type,
                              vector<uint32_t>& in_list);
   bool BuildCopyTrans(uint32_t req_type,
@@ -301,10 +315,15 @@ class RocmAsync : public BaseTest {
   uint32_t req_write_;
   uint32_t req_copy_bidir_;
   uint32_t req_copy_unidir_;
+  uint32_t req_copy_all_bidir_;
+  uint32_t req_copy_all_unidir_;
 
   // List used to store transactions per user request
   vector<async_trans_t> trans_list_;
 
+  // List used to store transactions involving Cpu-Gpu pools
+  vector<async_trans_t> matrix_trans_list_;
+
   // Variable to store argument number
 
   // Variable to store argument number

projects/rocr-runtime/rocrtst/samples/rocm_async/rocm_async_io.cpp

@@ -0,0 +1,16 @@
+
+#include "common.hpp"
+#include "rocm_async.hpp"
+
+#include <stdlib.h>
+#include <assert.h>
+#include <algorithm>
+#include <unistd.h>
+#include <cctype>
+#include <sstream>
+
+void RocmAsync::RunIOBenchmark(async_trans_t& trans) {
+
+  std::cout << "Unsupported Request - Read / Write" << std::endl;
+  exit(1);
+}

projects/rocr-runtime/rocrtst/samples/rocm_async/rocm_async_parse.cpp

@@ -126,13 +126,13 @@ void RocmAsync::ParseArguments() {
       // Enable Unidirectional copy among all valid pools
       case 'a':
         copy_all_uni = true;
-        req_copy_unidir_ = REQ_COPY_UNIDIR;
+        req_copy_all_unidir_ = REQ_COPY_ALL_UNIDIR;
         break;
 
       // Enable Bidirectional copy among all valid pools
       case 'A':
         copy_all_bi = true;
-        req_copy_bidir_ = REQ_COPY_BIDIR;
+        req_copy_all_bidir_ = REQ_COPY_ALL_BIDIR;
         break;
 
       // getopt implementation returns the value of the unknown
@@ -195,10 +195,17 @@ void RocmAsync::ParseArguments() {
   }
 
   // Initialize the list of buffer sizes to use in copy/read/write operations
+  // For All Copy operations use only one buffer size
   if (size_list_.size() == 0) {
     uint32_t size_len = sizeof(SIZE_LIST)/sizeof(uint32_t);
     for (uint32_t idx = 0; idx < size_len; idx++) {
-      size_list_.push_back(SIZE_LIST[idx]);
+      if ((copy_all_bi) || (copy_all_uni)) {
+        if (idx == 0) {
+          size_list_.push_back(SIZE_LIST[idx]);
+        }
+      } else {
+        size_list_.push_back(SIZE_LIST[idx]);
+      }
     }
   }
   std::sort(size_list_.begin(), size_list_.end());

projects/rocr-runtime/rocrtst/samples/rocm_async/rocm_async_print.cpp

@@ -19,14 +19,25 @@ void RocmAsync::PrintHelpScreen() {
   std::cout << "\t -a Perform Unidirectional Copy involving all pool combinations" << std::endl;
   std::cout << "\t -A Perform Bidirectional Copy involving all pool combinations" << std::endl;
   std::cout << std::endl;
-
-  std::cout << "\t @note 1: Removes copyReq(srcI, dstI) - where Src & Dst Pools are same" << std::endl;
+  
   std::cout << std::endl;
-  std::cout << "\t @note 2: Removes copyReq(srcI, dstJ) - where Src & Dst Pools are Cpu bound " << std::endl;
+  std::cout << "\t @note 1: Removes copyReq(srcI, dstJ) - where either Src or Dst Pool is fine-grained" << std::endl;
   std::cout << std::endl;
-  std::cout << "\t @note 3: Treats copyReq(dstI, srcJ) as NOT EQUAL to copyReq(dstJ, srcI) " << std::endl;
+  std::cout << "\t @note 2: Treats copyReq(dstI, srcJ) as NOT EQUAL to copyReq(dstJ, srcI) " << std::endl;
   std::cout << "\t            Underlying copy engine could be different " << std::endl;
   std::cout << std::endl;
+
+  /*
+  std::cout << "\t @note 1: Removes copyReq(srcI, dstI) - where Src & Dst Pools are same" << std::endl;
+  std::cout << std::endl;
+  std::cout << "\t @note 2: Removes copyReq(srcI, dstJ) - where Src & Dst Pools are Cpu bound" << std::endl;
+  std::cout << std::endl;
+  std::cout << "\t @note 3: Removes copyReq(srcI, dstJ) - where either Src or Dst Pool is fine-grained" << std::endl;
+  std::cout << std::endl;
+  std::cout << "\t @note 4: Treats copyReq(dstI, srcJ) as NOT EQUAL to copyReq(dstJ, srcI) " << std::endl;
+  std::cout << "\t            Underlying copy engine could be different " << std::endl;
+  std::cout << std::endl;
+  */
 }
 
 // @brief: Print the topology of Memory Pools and Agents present in system

projects/rocr-runtime/rocrtst/samples/rocm_async/rocm_async_report.cpp

@@ -100,6 +100,14 @@ void RocmAsync::Display() const {
     std::cout << std::endl;
     return;
   }
+
+  if ((req_copy_all_bidir_ == REQ_COPY_ALL_BIDIR) ||
+      (req_copy_all_unidir_ == REQ_COPY_ALL_UNIDIR)) {
+    DisplayCopyTimeMatrix();
+    std::cout << std::endl;
+    return;
+  }
+
   for (uint32_t idx = 0; idx < trans_size; idx++) {
     async_trans_t trans = trans_list_[idx];
     if ((trans.req_type_ == REQ_COPY_BIDIR) ||
@@ -120,10 +128,6 @@ void RocmAsync::DisplayIOTime(async_trans_t& trans) const {
 
 void RocmAsync::DisplayCopyTime(async_trans_t& trans) const {
   
-  // Get the frequency of Gpu Timestamping
-  uint64_t sys_freq = 0;
-  hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &sys_freq);
-  
   // Print Benchmark Header
   uint32_t src_idx = trans.copy.src_idx_;
   uint32_t dst_idx = trans.copy.dst_idx_;
@@ -133,35 +137,121 @@ void RocmAsync::DisplayCopyTime(async_trans_t& trans) const {
   hsa_device_type_t dst_dev_type = agent_list_[dst_dev_idx].device_type_;
   printCopyBanner(src_idx, src_dev_type, dst_idx, dst_dev_type);
   
-  double avg_time = 0;
-  double min_time = 0;
-  double bandwidth = 0;
-  uint32_t data_size = 0;
-  double peak_bandwidth = 0;
   uint32_t size_len = size_list_.size();
   for (uint32_t idx = 0; idx < size_len; idx++) {
-    
-    // Adjust size of data involved in copy
-    data_size = size_list_[idx];
-    if (trans.copy.bidir_ == true) {
-      data_size += size_list_[idx];
-    }
-    data_size = data_size * 1024 * 1024;
-
-    // Copy operation does not involve a Gpu device
-    if (trans.copy.uses_gpu_ != true) {
-      avg_time = trans.cpu_avg_time_[idx];
-      min_time = trans.cpu_min_time_[idx];
-      bandwidth = (double)data_size / avg_time / 1000 / 1000 / 1000;
-      peak_bandwidth = (double)data_size / min_time / 1000 / 1000 / 1000;
-    } else {
-      avg_time = trans.gpu_avg_time_[idx] / sys_freq;
-      min_time = trans.gpu_min_time_[idx] / sys_freq;
-      bandwidth = (double)data_size / avg_time / 1000 / 1000 / 1000;
-      peak_bandwidth = (double)data_size / min_time / 1000 / 1000 / 1000;
-    }
-
-    printRecord(size_list_[idx], avg_time, bandwidth, min_time, peak_bandwidth);
+    printRecord(size_list_[idx], trans.avg_time_[idx],
+                trans.avg_bandwidth_[idx], trans.min_time_[idx],
+                trans.peak_bandwidth_[idx]);
   }
 }
 
+void RocmAsync::DisplayCopyTimeMatrix() const {
+  
+  double* avg_matrix = new double[agent_index_ * agent_index_]();
+  double* peak_matrix = new double[agent_index_ * agent_index_]();
+  uint32_t trans_size = trans_list_.size();
+  for (uint32_t idx = 0; idx < trans_size; idx++) {
+    async_trans_t trans = trans_list_[idx];
+    uint32_t src_idx = trans.copy.src_idx_;
+    uint32_t dst_idx = trans.copy.dst_idx_;
+    uint32_t src_dev_idx = pool_list_[src_idx].agent_index_;
+    uint32_t dst_dev_idx = pool_list_[dst_idx].agent_index_;
+    avg_matrix[(src_dev_idx * agent_index_) + dst_dev_idx] = trans.avg_bandwidth_[0];
+    peak_matrix[(src_dev_idx * agent_index_) + dst_dev_idx] = trans.peak_bandwidth_[0];
+  }
+
+  uint32_t format = 12;
+  std::cout.setf(ios::left);
+
+  std::cout << std::endl;
+  std::cout.width(format);
+  std::cout << "";
+  std::cout.width(format);
+  if (req_copy_all_unidir_ == REQ_COPY_ALL_UNIDIR) {
+    std::cout << "Peak Bandwidth For Unidirectional Copies GB/sec";
+  } else {
+    std::cout << "Peak Bandwidth For Bidirectional Copies GB/sec";
+  }
+  std::cout << std::endl;
+  std::cout << std::endl;
+
+  std::cout.width(format);
+  std::cout << "";
+  std::cout.width(format);
+  std::cout << "";
+  for (uint32_t idx0 = 0; idx0 < agent_index_; idx0++) {
+    std::cout.width(format);
+    std::stringstream agent_id;
+    agent_id << "Dev-" << idx0;
+    std::cout << agent_id.str();
+  }
+  std::cout << std::endl;
+  std::cout << std::endl;
+  for (uint32_t idx0 = 0; idx0 < agent_index_; idx0++) {
+    std::cout.width(format);
+    std::cout << "";
+    std::stringstream agent_id;
+    agent_id << "Dev-" << idx0;
+    std::cout.width(format);
+    std::cout << agent_id.str();
+    for (uint32_t idx1 = 0; idx1 < agent_index_; idx1++) {
+      std::cout.width(format);
+      std::cout << peak_matrix[(idx0 * agent_index_) + idx1];
+    }
+    std::cout << std::endl;
+    std::cout << std::endl;
+  }
+  std::cout << std::endl;
+
+  std::cout.width(format);
+  std::cout << "";
+  std::cout.width(format);
+  if (req_copy_all_unidir_ == REQ_COPY_ALL_UNIDIR) {
+    std::cout << "Average Bandwidth For Unidirectional Copies GB/sec";
+  } else {
+    std::cout << "Average Bandwidth For Bidirectional Copies GB/sec";
+  }
+  std::cout << std::endl;
+  std::cout << std::endl;
+
+  std::cout.width(format);
+  std::cout << "";
+  std::cout.width(format);
+  std::cout << "";
+  for (uint32_t idx0 = 0; idx0 < agent_index_; idx0++) {
+    std::cout.width(format);
+    std::stringstream agent_id;
+    agent_id << "Dev-" << idx0;
+    std::cout << agent_id.str();
+  }
+  std::cout << std::endl;
+  std::cout << std::endl;
+  for (uint32_t idx0 = 0; idx0 < agent_index_; idx0++) {
+    std::cout.width(format);
+    std::cout << "";
+    std::stringstream agent_id;
+    agent_id << "Dev-" << idx0;
+    std::cout.width(format);
+    std::cout << agent_id.str();
+    for (uint32_t idx1 = 0; idx1 < agent_index_; idx1++) {
+      std::cout.width(format);
+      std::cout << avg_matrix[(idx0 * agent_index_) + idx1];
+    }
+    std::cout << std::endl;
+    std::cout << std::endl;
+  }
+  std::cout << std::endl;
+
+  /*
+  std::cout.width(format);
+  std::cout << "";
+  std::cout << "@note-1: ZERO in Dev-i != Dev-j means DIRECT PATH doesn't exist";
+  std::cout << std::endl;
+  std::cout.width(format);
+  std::cout << "";
+  std::cout << "@note-2: ZERO in Dev-i == Dev-j means COPY operation is filtered out";
+  std::cout << std::endl;
+  std::cout << std::endl;
+  */
+}
+

projects/rocr-runtime/rocrtst/samples/rocm_async/rocm_async_trans.cpp

@@ -67,6 +67,16 @@ bool RocmAsync::BuildCopyTrans(uint32_t req_type,
     uint32_t src_dev_idx = pool_list_[src_idx].agent_index_;
     hsa_device_type_t src_dev_type = agent_list_[src_dev_idx].device_type_;
 
+    // Determine if dst pool is fine grained, if so filter out
+    // the transaction
+    if ((req_type == REQ_COPY_ALL_BIDIR) ||
+        (req_type == REQ_COPY_ALL_UNIDIR)) {
+      bool src_fine_grained =  pool_list_[src_idx].is_fine_grained_;
+      if (src_fine_grained) {
+        continue;
+      }
+    }
+
     for (uint32_t jdx = 0; jdx < dst_size; jdx++) {
     
       // Retrieve Roc runtime handles for Dst memory pool and agents
@@ -76,16 +86,30 @@ bool RocmAsync::BuildCopyTrans(uint32_t req_type,
       uint32_t dst_dev_idx = pool_list_[dst_idx].agent_index_;
       hsa_device_type_t dst_dev_type = agent_list_[dst_dev_idx].device_type_;
 
+      // Determine if dst pool is fine grained, if so filter out
+      // the transaction
+      if ((req_type == REQ_COPY_ALL_BIDIR) ||
+          (req_type == REQ_COPY_ALL_UNIDIR)) {
+        bool dst_fine_grained =  pool_list_[dst_idx].is_fine_grained_;
+        if (dst_fine_grained) {
+          continue;
+        }
+      }
+
       // Filter out transaction when Src & Dst pools belong to Cpu
+      /*
       if ((src_dev_type == HSA_DEVICE_TYPE_CPU) &&
           (dst_dev_type == HSA_DEVICE_TYPE_CPU)) {
         continue;
       }
+      */
 
       // Filter out transaction with same Src & Dst pools
+      /*
       if (src_idx == dst_idx) {
         continue;
       }
+      */
       
       // Determine if accessibility to src pool for dst agent is not denied
       status = hsa_amd_agent_memory_pool_get_info(dst_agent, src_pool,
@@ -111,7 +135,8 @@ bool RocmAsync::BuildCopyTrans(uint32_t req_type,
       trans.copy.dst_idx_ = dst_idx;
       trans.copy.src_pool_ = src_pool;
       trans.copy.dst_pool_ = dst_pool;
-      trans.copy.bidir_ = (req_type == REQ_COPY_BIDIR);
+      trans.copy.bidir_ = ((req_type == REQ_COPY_BIDIR) ||
+                           (req_type == REQ_COPY_ALL_BIDIR));
       trans.copy.uses_gpu_ = ((src_dev_type == HSA_DEVICE_TYPE_GPU) ||
                               (dst_dev_type == HSA_DEVICE_TYPE_GPU));
       trans_list_.push_back(trans);
@@ -128,6 +153,14 @@ bool RocmAsync::BuildUnidirCopyTrans() {
   return BuildCopyTrans(REQ_COPY_UNIDIR, src_list_, dst_list_);
 }
 
+bool RocmAsync::BuildAllPoolsBidirCopyTrans() {
+  return BuildCopyTrans(REQ_COPY_ALL_BIDIR, bidir_list_, bidir_list_);
+}
+
+bool RocmAsync::BuildAllPoolsUnidirCopyTrans() {
+  return BuildCopyTrans(REQ_COPY_ALL_UNIDIR, src_list_, dst_list_);
+}
+
 // @brief: Builds a list of transaction per user request
 bool RocmAsync::BuildTransList() {
   
@@ -167,7 +200,66 @@ bool RocmAsync::BuildTransList() {
     }
   }
 
+  // Build list of All Bidir Copy transactions per user request
+  status = false;
+  if (req_copy_all_bidir_ == REQ_COPY_ALL_BIDIR) {
+    status = BuildAllPoolsBidirCopyTrans();
+    if (status == false) {
+      return status;
+    }
+  }
+
+  // Build list of All Unidir Copy transactions per user request
+  status = false;
+  if (req_copy_all_unidir_ == REQ_COPY_ALL_UNIDIR) {
+    status = BuildAllPoolsUnidirCopyTrans();
+    if (status == false) {
+      return status;
+    }
+  }
+
   // All of the transaction are built up
   return true;
 }
 
+void RocmAsync::ComputeCopyTime(async_trans_t& trans) {
+
+  // Get the frequency of Gpu Timestamping
+  uint64_t sys_freq = 0;
+  hsa_system_get_info(HSA_SYSTEM_INFO_TIMESTAMP_FREQUENCY, &sys_freq);
+  
+  double avg_time = 0;
+  double min_time = 0;
+  double bandwidth = 0;
+  uint32_t data_size = 0;
+  double peak_bandwidth = 0;
+  uint32_t size_len = size_list_.size();
+  for (uint32_t idx = 0; idx < size_len; idx++) {
+    
+    // Adjust size of data involved in copy
+    data_size = size_list_[idx];
+    if (trans.copy.bidir_ == true) {
+      data_size += size_list_[idx];
+    }
+    data_size = data_size * 1024 * 1024;
+
+    // Copy operation does not involve a Gpu device
+    if (trans.copy.uses_gpu_ != true) {
+      avg_time = trans.cpu_avg_time_[idx];
+      min_time = trans.cpu_min_time_[idx];
+      bandwidth = (double)data_size / avg_time / 1000 / 1000 / 1000;
+      peak_bandwidth = (double)data_size / min_time / 1000 / 1000 / 1000;
+    } else {
+      avg_time = trans.gpu_avg_time_[idx] / sys_freq;
+      min_time = trans.gpu_min_time_[idx] / sys_freq;
+      bandwidth = (double)data_size / avg_time / 1000 / 1000 / 1000;
+      peak_bandwidth = (double)data_size / min_time / 1000 / 1000 / 1000;
+    }
+
+    trans.min_time_.push_back(min_time);
+    trans.avg_time_.push_back(avg_time);
+    trans.avg_bandwidth_.push_back(bandwidth);
+    trans.peak_bandwidth_.push_back(peak_bandwidth);
+  }
+}
+

projects/rocr-runtime/rocrtst/samples/rocm_async/rocm_async_validate.cpp

@@ -130,9 +130,11 @@ bool RocmAsync::ValidateArguments() {
 
   // Determine if user has requested a Copy
   // operation that is bidirectional and gave
-  // valid inputs
+  // valid inputs. Same validation is applied
+  // for all-to-all unidirectional copy operation
   status = false;
-  if (req_copy_bidir_ == REQ_COPY_BIDIR) {
+  if ((req_copy_bidir_ == REQ_COPY_BIDIR) ||
+      (req_copy_all_bidir_ == REQ_COPY_ALL_BIDIR)) {
     status = ValidateBidirCopyReq();
     if (status == false) {
       return status;
@@ -141,9 +143,11 @@ bool RocmAsync::ValidateArguments() {
 
   // Determine if user has requested a Copy
   // operation that is unidirectional and gave
-  // valid inputs
+  // valid inputs. Same validation is applied
+  // for all-to-all bidirectional copy operation
   status = false;
-  if (req_copy_unidir_ == REQ_COPY_UNIDIR) {
+  if ((req_copy_unidir_ == REQ_COPY_UNIDIR) ||
+      (req_copy_all_unidir_ == REQ_COPY_ALL_UNIDIR)) {
     status = ValidateUnidirCopyReq();
     if (status == false) {
       return status;