Update MP UT to support arbitrary # of GPUs; multiple bugfixes (#16)

* Fixing temp file creation/deletion for Clique kernel mode.

* Refactoring of MP unit tests; include bugfixes and general support for any number of GPUs

* GroupCall MP UT properly quits when too many devices specified

* MP UT will programmatically set NCCL_COMM_ID if not specified; updated install script

[ROCm/rccl commit: d00b7d17bd]
This commit is contained in:
Stanley Tsang
2021-02-05 17:49:25 -07:00
committed by GitHub
parent fe8923ebba
commit f152c8d160
23 changed files with 538 additions and 716 deletions
@@ -23,12 +23,16 @@ namespace CorrectnessTests
}
}
void TestScatter(int rank, Dataset& dataset)
void TestScatter(int rank, Dataset& dataset, bool& pass)
{
// Prepare input / output / expected results
SetUpPerProcess(rank, ncclCollScatter, comms[rank], streams[rank], dataset);
if (numDevices > numDevicesAvailable) return;
if (numDevices > numDevicesAvailable)
{
pass = true;
return;
}
Barrier barrier(rank, numDevices, std::atoi(getenv("NCCL_COMM_ID")));
@@ -50,7 +54,7 @@ namespace CorrectnessTests
HIP_CALL(hipStreamSynchronize(streams[rank]));
// Check results
ValidateResults(dataset, rank);
pass = ValidateResults(dataset, rank);
barrier.Wait();
}