Removing OpenMP from unit tests (#163)

[ROCm/rccl commit: 000bce6f27]

projects/rccl/test/CMakeLists.txt

@@ -10,13 +10,6 @@ if(BUILD_TESTS)
       message(FATAL_ERROR "chrpath is required for UnitTests. Please install (e.g. sudo apt-get install chrpath)")
   endif()
 
-  # OpenMP is used to drive GPUs (one per thread)
-  if(EXISTS /etc/redhat-release)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp=libgomp -pthread")
-  else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -pthread")
-  endif()    
-
   # Download and unpack googletest at configure time
   configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt)
   execute_process(

projects/rccl/test/test_AllGather.cpp

@@ -4,7 +4,6 @@
  * See LICENSE.txt for license information
  ************************************************************************/
 #include "test_AllGather.hpp"
-#include <omp.h>
 
 namespace CorrectnessTests
 {
@@ -23,13 +22,14 @@ namespace CorrectnessTests
         size_t const sendCount = dataset.numElements / dataset.numDevices;
 
         // Launch the reduction (1 thread per GPU)
-        #pragma omp parallel for num_threads(numDevices)
+        ncclGroupStart();
         for (int i = 0; i < numDevices; i++)
         {
             ncclAllGather((int8_t *)dataset.inputs[i] + (i * byteCount),
                           dataset.outputs[i], sendCount,
                           dataType, comms[i], streams[i]);
         }
+        ncclGroupEnd();
 
         // Wait for reduction to complete
         Synchronize();
@@ -68,13 +68,14 @@ namespace CorrectnessTests
                 size_t const sendCount = subDataset.numElements / subDataset.numDevices;
 
                 // Launch the reduction (1 thread per GPU)
-                #pragma omp parallel for num_threads(numDevices)
+                ncclGroupStart();
                 for (int i = 0; i < numDevices; i++)
                 {
                     ncclAllGather((int8_t *)subDataset.inputs[i] + (i * byteCount),
                                   subDataset.outputs[i], sendCount,
                                   dataType, comms[i], streams[i]);
                 }
+                ncclGroupEnd();
 
                 // Wait for reduction to complete
                 Synchronize();

projects/rccl/test/test_AllReduce.cpp

@@ -5,7 +5,6 @@
  ************************************************************************/
 
 #include "test_AllReduce.hpp"
-#include <omp.h>
 
 namespace CorrectnessTests
 {
@@ -20,12 +19,13 @@ namespace CorrectnessTests
         ComputeExpectedResults(dataset, op);
 
         // Launch the reduction (1 thread per GPU)
-        #pragma omp parallel for num_threads(numDevices)
+        ncclGroupStart();
         for (int i = 0; i < numDevices; i++)
         {
             ncclAllReduce(dataset.inputs[i], dataset.outputs[i],
                           numElements, dataType, op, comms[i], streams[i]);
         }
+        ncclGroupEnd();
 
         // Wait for reduction to complete
         Synchronize();

projects/rccl/test/test_AllReduceAbort.cpp

@@ -6,7 +6,6 @@
 
 #include "test_AllReduceAbort.hpp"
 #include "../include/core.h"
-#include <omp.h>
 
 #define NUM_ITER 8
 #define FAKE_OP_COUNT NUM_ITER+1

projects/rccl/test/test_Broadcast.cpp

@@ -5,7 +5,6 @@
  ************************************************************************/
 
 #include "test_Broadcast.hpp"
-#include <omp.h>
 
 namespace CorrectnessTests
 {
@@ -25,7 +24,7 @@ namespace CorrectnessTests
             ComputeExpectedResults(dataset, root);
 
             // Launch the reduction (1 thread per GPU)
-            #pragma omp parallel for num_threads(numDevices)
+            ncclGroupStart();
             for (int i = 0; i < numDevices; i++)
             {
                 ncclBroadcast(dataset.inputs[i],
@@ -33,7 +32,7 @@ namespace CorrectnessTests
                               numElements, dataType,
                               root, comms[i], streams[i]);
             }
-
+            ncclGroupEnd();
 
             // Wait for reduction to complete
             Synchronize();

projects/rccl/test/test_Broadcast.hpp

@@ -7,7 +7,6 @@
 #define TEST_BROADCAST_HPP
 
 #include "CorrectnessTest.hpp"
-#include <omp.h>
 
 namespace CorrectnessTests
 {

projects/rccl/test/test_BroadcastAbort.cpp

@@ -6,7 +6,6 @@
 
 #include "test_BroadcastAbort.hpp"
 #include "../include/core.h"
-#include <omp.h>
 
 #define NUM_ITER 8
 #define FAKE_OP_COUNT NUM_ITER+1

projects/rccl/test/test_CombinedCalls.cpp

@@ -11,8 +11,6 @@
 #include "test_Reduce.hpp"
 #include "test_ReduceScatter.hpp"
 
-#include <omp.h>
-
 namespace CorrectnessTests
 {
     TEST_P(CombinedCallsCorrectnessTest, Correctness)
@@ -38,7 +36,7 @@ namespace CorrectnessTests
         size_t const byteCount = datasets[0].NumBytes() / numDevices;
         size_t const elemCount = numElements / numDevices;
 
-        #pragma omp parallel for num_threads(numDevices)
+        ncclGroupStart();
         for (int i = 0; i < numDevices; i++)
         {
             ncclAllGather((int8_t *)datasets[0].inputs[i] + (i * byteCount),
@@ -63,6 +61,7 @@ namespace CorrectnessTests
                               elemCount, dataType, op,
                               comms[i], streams[i]);
         }
+        ncclGroupEnd();
 
         // Wait for reduction to complete
         Synchronize();

projects/rccl/test/test_GroupCalls.cpp

@@ -11,8 +11,6 @@
 #include "test_Reduce.hpp"
 #include "test_ReduceScatter.hpp"
 
-#include <omp.h>
-
 namespace CorrectnessTests
 {
     TEST_P(GroupCallsCorrectnessTest, Correctness)

projects/rccl/test/test_Reduce.cpp

@@ -5,7 +5,6 @@
  ************************************************************************/
 
 #include "test_Reduce.hpp"
-#include <omp.h>
 
 namespace CorrectnessTests
 {
@@ -25,7 +24,7 @@ namespace CorrectnessTests
             ComputeExpectedResults(dataset, op, root);
 
             // Launch the reduction (1 thread per GPU)
-            #pragma omp parallel for num_threads(numDevices)
+            ncclGroupStart();
             for (int i = 0; i < numDevices; i++)
             {
                 ncclReduce(dataset.inputs[i],
@@ -33,6 +32,7 @@ namespace CorrectnessTests
                            numElements, dataType, op,
                            root, comms[i], streams[i]);
             }
+            ncclGroupEnd();
 
             // Wait for reduction to complete
             Synchronize();

projects/rccl/test/test_ReduceScatter.cpp

@@ -5,7 +5,6 @@
  ************************************************************************/
 
 #include "test_ReduceScatter.hpp"
-#include <omp.h>
 
 namespace CorrectnessTests
 {
@@ -24,7 +23,7 @@ namespace CorrectnessTests
         size_t const recvCount = dataset.numElements / dataset.numDevices;
 
         // Launch the reduction (1 thread per GPU)
-        #pragma omp parallel for num_threads(numDevices)
+        ncclGroupStart();
         for (int i = 0; i < numDevices; i++)
         {
             ncclReduceScatter(dataset.inputs[i],
@@ -32,7 +31,7 @@ namespace CorrectnessTests
                               recvCount, dataType, op,
                               comms[i], streams[i]);
         }
-
+        ncclGroupEnd();
 
         // Wait for reduction to complete
         Synchronize();