From 71635198b843c4b5797ab004685e20bb36d1c856 Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Fri, 20 Dec 2019 11:41:56 -0700 Subject: [PATCH] Removing OpenMP from unit tests (#163) [ROCm/rccl commit: 000bce6f27b0689ce7f5d2d9b22d9b718548aee9] --- projects/rccl/test/CMakeLists.txt | 7 ------- projects/rccl/test/test_AllGather.cpp | 7 ++++--- projects/rccl/test/test_AllReduce.cpp | 4 ++-- projects/rccl/test/test_AllReduceAbort.cpp | 1 - projects/rccl/test/test_Broadcast.cpp | 5 ++--- projects/rccl/test/test_Broadcast.hpp | 1 - projects/rccl/test/test_BroadcastAbort.cpp | 1 - projects/rccl/test/test_CombinedCalls.cpp | 5 ++--- projects/rccl/test/test_GroupCalls.cpp | 2 -- projects/rccl/test/test_Reduce.cpp | 4 ++-- projects/rccl/test/test_ReduceScatter.cpp | 5 ++--- 11 files changed, 14 insertions(+), 28 deletions(-) diff --git a/projects/rccl/test/CMakeLists.txt b/projects/rccl/test/CMakeLists.txt index 66236c2f18..ee542902b6 100644 --- a/projects/rccl/test/CMakeLists.txt +++ b/projects/rccl/test/CMakeLists.txt @@ -10,13 +10,6 @@ if(BUILD_TESTS) message(FATAL_ERROR "chrpath is required for UnitTests. Please install (e.g. sudo apt-get install chrpath)") endif() - # OpenMP is used to drive GPUs (one per thread) - if(EXISTS /etc/redhat-release) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp=libgomp -pthread") - else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -pthread") - endif() - # Download and unpack googletest at configure time configure_file(CMakeLists.txt.in googletest-download/CMakeLists.txt) execute_process( diff --git a/projects/rccl/test/test_AllGather.cpp b/projects/rccl/test/test_AllGather.cpp index b28df48e87..df5aea23e3 100644 --- a/projects/rccl/test/test_AllGather.cpp +++ b/projects/rccl/test/test_AllGather.cpp @@ -4,7 +4,6 @@ * See LICENSE.txt for license information ************************************************************************/ #include "test_AllGather.hpp" -#include namespace CorrectnessTests { @@ -23,13 +22,14 @@ namespace CorrectnessTests size_t const sendCount = dataset.numElements / dataset.numDevices; // Launch the reduction (1 thread per GPU) - #pragma omp parallel for num_threads(numDevices) + ncclGroupStart(); for (int i = 0; i < numDevices; i++) { ncclAllGather((int8_t *)dataset.inputs[i] + (i * byteCount), dataset.outputs[i], sendCount, dataType, comms[i], streams[i]); } + ncclGroupEnd(); // Wait for reduction to complete Synchronize(); @@ -68,13 +68,14 @@ namespace CorrectnessTests size_t const sendCount = subDataset.numElements / subDataset.numDevices; // Launch the reduction (1 thread per GPU) - #pragma omp parallel for num_threads(numDevices) + ncclGroupStart(); for (int i = 0; i < numDevices; i++) { ncclAllGather((int8_t *)subDataset.inputs[i] + (i * byteCount), subDataset.outputs[i], sendCount, dataType, comms[i], streams[i]); } + ncclGroupEnd(); // Wait for reduction to complete Synchronize(); diff --git a/projects/rccl/test/test_AllReduce.cpp b/projects/rccl/test/test_AllReduce.cpp index 0fd5eedf91..86d4654b7b 100644 --- a/projects/rccl/test/test_AllReduce.cpp +++ b/projects/rccl/test/test_AllReduce.cpp @@ -5,7 +5,6 @@ ************************************************************************/ #include "test_AllReduce.hpp" -#include namespace CorrectnessTests { @@ -20,12 +19,13 @@ namespace CorrectnessTests ComputeExpectedResults(dataset, op); // Launch the reduction (1 thread per GPU) - #pragma omp parallel for num_threads(numDevices) + ncclGroupStart(); for (int i = 0; i < numDevices; i++) { ncclAllReduce(dataset.inputs[i], dataset.outputs[i], numElements, dataType, op, comms[i], streams[i]); } + ncclGroupEnd(); // Wait for reduction to complete Synchronize(); diff --git a/projects/rccl/test/test_AllReduceAbort.cpp b/projects/rccl/test/test_AllReduceAbort.cpp index 9400bd84fc..cccca4a547 100644 --- a/projects/rccl/test/test_AllReduceAbort.cpp +++ b/projects/rccl/test/test_AllReduceAbort.cpp @@ -6,7 +6,6 @@ #include "test_AllReduceAbort.hpp" #include "../include/core.h" -#include #define NUM_ITER 8 #define FAKE_OP_COUNT NUM_ITER+1 diff --git a/projects/rccl/test/test_Broadcast.cpp b/projects/rccl/test/test_Broadcast.cpp index 0e728b0153..a98d5884cb 100644 --- a/projects/rccl/test/test_Broadcast.cpp +++ b/projects/rccl/test/test_Broadcast.cpp @@ -5,7 +5,6 @@ ************************************************************************/ #include "test_Broadcast.hpp" -#include namespace CorrectnessTests { @@ -25,7 +24,7 @@ namespace CorrectnessTests ComputeExpectedResults(dataset, root); // Launch the reduction (1 thread per GPU) - #pragma omp parallel for num_threads(numDevices) + ncclGroupStart(); for (int i = 0; i < numDevices; i++) { ncclBroadcast(dataset.inputs[i], @@ -33,7 +32,7 @@ namespace CorrectnessTests numElements, dataType, root, comms[i], streams[i]); } - + ncclGroupEnd(); // Wait for reduction to complete Synchronize(); diff --git a/projects/rccl/test/test_Broadcast.hpp b/projects/rccl/test/test_Broadcast.hpp index 5ccf351592..358d8cf09a 100644 --- a/projects/rccl/test/test_Broadcast.hpp +++ b/projects/rccl/test/test_Broadcast.hpp @@ -7,7 +7,6 @@ #define TEST_BROADCAST_HPP #include "CorrectnessTest.hpp" -#include namespace CorrectnessTests { diff --git a/projects/rccl/test/test_BroadcastAbort.cpp b/projects/rccl/test/test_BroadcastAbort.cpp index 28596cc52a..824bced360 100644 --- a/projects/rccl/test/test_BroadcastAbort.cpp +++ b/projects/rccl/test/test_BroadcastAbort.cpp @@ -6,7 +6,6 @@ #include "test_BroadcastAbort.hpp" #include "../include/core.h" -#include #define NUM_ITER 8 #define FAKE_OP_COUNT NUM_ITER+1 diff --git a/projects/rccl/test/test_CombinedCalls.cpp b/projects/rccl/test/test_CombinedCalls.cpp index 4b51ab5375..41e93c78a2 100644 --- a/projects/rccl/test/test_CombinedCalls.cpp +++ b/projects/rccl/test/test_CombinedCalls.cpp @@ -11,8 +11,6 @@ #include "test_Reduce.hpp" #include "test_ReduceScatter.hpp" -#include - namespace CorrectnessTests { TEST_P(CombinedCallsCorrectnessTest, Correctness) @@ -38,7 +36,7 @@ namespace CorrectnessTests size_t const byteCount = datasets[0].NumBytes() / numDevices; size_t const elemCount = numElements / numDevices; - #pragma omp parallel for num_threads(numDevices) + ncclGroupStart(); for (int i = 0; i < numDevices; i++) { ncclAllGather((int8_t *)datasets[0].inputs[i] + (i * byteCount), @@ -63,6 +61,7 @@ namespace CorrectnessTests elemCount, dataType, op, comms[i], streams[i]); } + ncclGroupEnd(); // Wait for reduction to complete Synchronize(); diff --git a/projects/rccl/test/test_GroupCalls.cpp b/projects/rccl/test/test_GroupCalls.cpp index 77780b633d..3cfd55c6f5 100644 --- a/projects/rccl/test/test_GroupCalls.cpp +++ b/projects/rccl/test/test_GroupCalls.cpp @@ -11,8 +11,6 @@ #include "test_Reduce.hpp" #include "test_ReduceScatter.hpp" -#include - namespace CorrectnessTests { TEST_P(GroupCallsCorrectnessTest, Correctness) diff --git a/projects/rccl/test/test_Reduce.cpp b/projects/rccl/test/test_Reduce.cpp index 9844e928c1..8234f2ab05 100644 --- a/projects/rccl/test/test_Reduce.cpp +++ b/projects/rccl/test/test_Reduce.cpp @@ -5,7 +5,6 @@ ************************************************************************/ #include "test_Reduce.hpp" -#include namespace CorrectnessTests { @@ -25,7 +24,7 @@ namespace CorrectnessTests ComputeExpectedResults(dataset, op, root); // Launch the reduction (1 thread per GPU) - #pragma omp parallel for num_threads(numDevices) + ncclGroupStart(); for (int i = 0; i < numDevices; i++) { ncclReduce(dataset.inputs[i], @@ -33,6 +32,7 @@ namespace CorrectnessTests numElements, dataType, op, root, comms[i], streams[i]); } + ncclGroupEnd(); // Wait for reduction to complete Synchronize(); diff --git a/projects/rccl/test/test_ReduceScatter.cpp b/projects/rccl/test/test_ReduceScatter.cpp index 11007732b5..f6461f20b8 100644 --- a/projects/rccl/test/test_ReduceScatter.cpp +++ b/projects/rccl/test/test_ReduceScatter.cpp @@ -5,7 +5,6 @@ ************************************************************************/ #include "test_ReduceScatter.hpp" -#include namespace CorrectnessTests { @@ -24,7 +23,7 @@ namespace CorrectnessTests size_t const recvCount = dataset.numElements / dataset.numDevices; // Launch the reduction (1 thread per GPU) - #pragma omp parallel for num_threads(numDevices) + ncclGroupStart(); for (int i = 0; i < numDevices; i++) { ncclReduceScatter(dataset.inputs[i], @@ -32,7 +31,7 @@ namespace CorrectnessTests recvCount, dataType, op, comms[i], streams[i]); } - + ncclGroupEnd(); // Wait for reduction to complete Synchronize();