From f6f5e16fe6881776932fd6838907e32cbd0e6ebd Mon Sep 17 00:00:00 2001 From: Stanley Tsang Date: Thu, 10 Jun 2021 14:35:39 -0600 Subject: [PATCH] Fixing bug with ExtractSubDataset function not fully initializing subdataset (#390) --- test/CorrectnessTest.hpp | 1 + test/test_AllGather.cpp | 4 ++-- test/test_AllReduce.cpp | 2 +- test/test_AllReduceAbort.cpp | 2 +- test/test_AllReduceGroup.cpp | 6 +++--- test/test_Broadcast.cpp | 2 +- test/test_BroadcastAbort.cpp | 2 +- test/test_CombinedCalls.cpp | 9 ++++++++- test/test_GroupCalls.cpp | 9 ++++++++- test/test_Reduce.cpp | 2 +- test/test_ReduceScatter.cpp | 2 +- 11 files changed, 28 insertions(+), 13 deletions(-) diff --git a/test/CorrectnessTest.hpp b/test/CorrectnessTest.hpp index ed7cfb44e9..b294e607ee 100644 --- a/test/CorrectnessTest.hpp +++ b/test/CorrectnessTest.hpp @@ -268,6 +268,7 @@ namespace CorrectnessTests subDataset.numElements = lastElement - startElement + 1; subDataset.dataType = dataType; subDataset.inPlace = inPlace; + subDataset.function = function; subDataset.inputs.resize(numDevices); subDataset.outputs.resize(numDevices); diff --git a/test/test_AllGather.cpp b/test/test_AllGather.cpp index d53ab4354d..295892bbd8 100644 --- a/test/test_AllGather.cpp +++ b/test/test_AllGather.cpp @@ -14,7 +14,7 @@ namespace CorrectnessTests // Prepare input / output / expected results Dataset dataset; - dataset.Initialize(numDevices, numElements, dataType, inPlace); + dataset.Initialize(numDevices, numElements, dataType, inPlace, ncclCollAllGather); FillDatasetWithPattern(dataset); ComputeExpectedResults(dataset); @@ -46,7 +46,7 @@ namespace CorrectnessTests // Allocate dataset Dataset dataset; - dataset.Initialize(numDevices, numElements, dataType, inPlace); + dataset.Initialize(numDevices, numElements, dataType, inPlace, ncclCollAllGather); // Loop over several offsets (so that device pointers are not aligned) for (int firstElement = 1; firstElement <= 11; firstElement += 2) diff --git a/test/test_AllReduce.cpp b/test/test_AllReduce.cpp index 1ef7b410e5..74e4017c19 100644 --- a/test/test_AllReduce.cpp +++ b/test/test_AllReduce.cpp @@ -14,7 +14,7 @@ namespace CorrectnessTests // Prepare input / output / expected results Dataset dataset; - dataset.Initialize(numDevices, numElements, dataType, inPlace); + dataset.Initialize(numDevices, numElements, dataType, inPlace, ncclCollAllReduce); FillDatasetWithPattern(dataset); ComputeExpectedResults(dataset, op); diff --git a/test/test_AllReduceAbort.cpp b/test/test_AllReduceAbort.cpp index e91dcfa0b9..64e34032ee 100644 --- a/test/test_AllReduceAbort.cpp +++ b/test/test_AllReduceAbort.cpp @@ -30,7 +30,7 @@ namespace CorrectnessTests // Prepare input / output / expected results Dataset dataset; - dataset.Initialize(numDevices, numElements, dataType, inPlace); + dataset.Initialize(numDevices, numElements, dataType, inPlace, ncclCollAllReduce); FillDatasetWithPattern(dataset); int gpu = 0; // GPU number to trigger abort diff --git a/test/test_AllReduceGroup.cpp b/test/test_AllReduceGroup.cpp index 5b4cbad4e5..2e86cbaf5a 100644 --- a/test/test_AllReduceGroup.cpp +++ b/test/test_AllReduceGroup.cpp @@ -15,9 +15,9 @@ namespace CorrectnessTests // Prepare input / output / expected results Dataset dataset1, dataset2, dataset3; - dataset1.Initialize(numDevices, numElements, dataType, inPlace); - dataset2.Initialize(numDevices, numElements, dataType, inPlace); - dataset3.Initialize(numDevices, numElements, dataType, inPlace); + dataset1.Initialize(numDevices, numElements, dataType, inPlace, ncclCollAllReduce); + dataset2.Initialize(numDevices, numElements, dataType, inPlace, ncclCollAllReduce); + dataset3.Initialize(numDevices, numElements, dataType, inPlace, ncclCollAllReduce); FillDatasetWithPattern(dataset1); FillDatasetWithPattern(dataset2); FillDatasetWithPattern(dataset3); diff --git a/test/test_Broadcast.cpp b/test/test_Broadcast.cpp index 955c2c1255..d273a67780 100644 --- a/test/test_Broadcast.cpp +++ b/test/test_Broadcast.cpp @@ -14,7 +14,7 @@ namespace CorrectnessTests // Allocate data Dataset dataset; - dataset.Initialize(numDevices, numElements, dataType, inPlace); + dataset.Initialize(numDevices, numElements, dataType, inPlace, ncclCollBroadcast); // Test each possible root for (int root = 0; root < numDevices; root++) diff --git a/test/test_BroadcastAbort.cpp b/test/test_BroadcastAbort.cpp index e7b3dfd07d..2f0d46a147 100644 --- a/test/test_BroadcastAbort.cpp +++ b/test/test_BroadcastAbort.cpp @@ -30,7 +30,7 @@ namespace CorrectnessTests // Prepare input / output / expected results Dataset dataset; - dataset.Initialize(numDevices, numElements, dataType, inPlace); + dataset.Initialize(numDevices, numElements, dataType, inPlace, ncclCollBroadcast); FillDatasetWithPattern(dataset); int root = 0; diff --git a/test/test_CombinedCalls.cpp b/test/test_CombinedCalls.cpp index ac980b2bef..851bc5acc8 100644 --- a/test/test_CombinedCalls.cpp +++ b/test/test_CombinedCalls.cpp @@ -20,9 +20,16 @@ namespace CorrectnessTests // Create multiple datasets for combined operation std::vector datasets(5); + std::vector ncclFuncs(5); + ncclFuncs.push_back(ncclCollAllGather); + ncclFuncs.push_back(ncclCollAllReduce); + ncclFuncs.push_back(ncclCollBroadcast); + ncclFuncs.push_back(ncclCollReduce); + ncclFuncs.push_back(ncclCollReduceScatter); + for (int i = 0; i < datasets.size(); i++) { - datasets[i].Initialize(numDevices, numElements, dataType, inPlace); + datasets[i].Initialize(numDevices, numElements, dataType, inPlace, ncclFuncs[i]); FillDatasetWithPattern(datasets[i]); } diff --git a/test/test_GroupCalls.cpp b/test/test_GroupCalls.cpp index f94dd6cf6a..1199846477 100644 --- a/test/test_GroupCalls.cpp +++ b/test/test_GroupCalls.cpp @@ -19,9 +19,16 @@ namespace CorrectnessTests // Create multiple datasets for group operation std::vector datasets(5); + std::vector ncclFuncs(5); + ncclFuncs.push_back(ncclCollAllGather); + ncclFuncs.push_back(ncclCollAllReduce); + ncclFuncs.push_back(ncclCollBroadcast); + ncclFuncs.push_back(ncclCollReduce); + ncclFuncs.push_back(ncclCollReduceScatter); + for (int i = 0; i < datasets.size(); i++) { - datasets[i].Initialize(numDevices, numElements, dataType, inPlace); + datasets[i].Initialize(numDevices, numElements, dataType, inPlace, ncclFuncs[i]); FillDatasetWithPattern(datasets[i]); } diff --git a/test/test_Reduce.cpp b/test/test_Reduce.cpp index 5405f3e0fc..c433bffe87 100644 --- a/test/test_Reduce.cpp +++ b/test/test_Reduce.cpp @@ -14,7 +14,7 @@ namespace CorrectnessTests // Allocate data Dataset dataset; - dataset.Initialize(numDevices, numElements, dataType, inPlace); + dataset.Initialize(numDevices, numElements, dataType, inPlace, ncclCollReduce); // Test each possible root for (int root = 0; root < numDevices; root++) diff --git a/test/test_ReduceScatter.cpp b/test/test_ReduceScatter.cpp index 26e62170ff..9979e29954 100644 --- a/test/test_ReduceScatter.cpp +++ b/test/test_ReduceScatter.cpp @@ -15,7 +15,7 @@ namespace CorrectnessTests // Prepare input / output / expected results Dataset dataset; - dataset.Initialize(numDevices, numElements, dataType, inPlace); + dataset.Initialize(numDevices, numElements, dataType, inPlace, ncclCollReduceScatter); FillDatasetWithPattern(dataset); ComputeExpectedResults(dataset, op);