AI/rocm-systems
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

UnitTest: add test cases for 2.14 API (ncclCommInitRankConfig and ncclCommFinalize for non-blocking communicator) (#662)

Browse Source
This commit is contained in:
Pedram Alizadeh
2022-12-13 16:05:09 -05:00
committed by GitHub
parent adafc0f759
commit 8250092367
6 changed files with 146 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/common/TestBedChild.cpp
+67 -2
View File
@@ -20,6 +20,26 @@
} \
}
#define CHILD_NCCL_CALL_NON_BLOCKING(msg) \
{ \
for (int i = 0; i < this->comms.size(); ++i) \
{ \
ncclResult_t ncclAsyncErr; \
int loop_counter = 0; \
do \
{ \
loop_counter++; \
if (loop_counter == MAX_LOOP_COUNTER) break; \
ncclCommGetAsyncError(this->comms[i], &ncclAsyncErr); \
} while(ncclAsyncErr == ncclInProgress); \
if (ncclAsyncErr != ncclSuccess) \
{ \
ERROR("Child process %d fails NCCL call %s with code %d\n", this->childId, msg, ncclAsyncErr); \
return TEST_FAIL; \
} \
} \
}
#define PIPE_READ(val) \
if (read(childReadFd, &val, sizeof(val)) != sizeof(val)) return TEST_FAIL;
@@ -126,6 +146,7 @@ namespace RcclUnitTesting
PIPE_READ(this->totalRanks);
PIPE_READ(this->rankOffset);
PIPE_READ(this->numCollectivesInGroup);
PIPE_READ(this->useBlocking);
bool useMultiRankPerGpu;
PIPE_READ(useMultiRankPerGpu);
@@ -177,6 +198,18 @@ namespace RcclUnitTesting
break;
}
}
else if (this->useBlocking == false)
{
// When non-blocking communicator is desired call ncclCommInitRankConfig with appropriate flag
ncclConfig_t config = NCCL_CONFIG_INITIALIZER;
config.blocking = 0;
if (ncclCommInitRankConfig(&this->comms[localRank], this->totalRanks, id, globalRank, &config) != ncclSuccess)
{
ERROR("Rank %d on child %d unable to call ncclCommInitRankConfig\n", globalRank, this->childId);
status = TEST_FAIL;
break;
}
}
else
{
if (ncclCommInitRank(&this->comms[localRank], this->totalRanks, id, globalRank) != ncclSuccess)
@@ -187,10 +220,26 @@ namespace RcclUnitTesting
}
}
}
if (status == TEST_SUCCESS)
if (this->useBlocking == false)
{
CHILD_NCCL_CALL(ncclGroupEnd(), "ncclGroupStart");
CHILD_NCCL_CALL_NON_BLOCKING("ncclCommGetAsyncErrorInitRankConfig");
}
if (status == TEST_SUCCESS)
{
// Check if the communicator is non-blocking
if (this->useBlocking == false)
{
// handle the ncclGroupEnd in case of non-blocking communication
ncclResult_t Group_End_state = ncclGroupEnd();
if (Group_End_state != ncclSuccess) CHILD_NCCL_CALL_NON_BLOCKING("ncclCommGetAsyncErrorGroup");
}
else
{
// In case of blocking communication just call ncclGroupEnd
CHILD_NCCL_CALL(ncclGroupEnd(), "ncclGroupEnd");
}
}
if (this->verbose) INFO("Child %d finishes InitComms() [%s]\n",
this->childId, status == TEST_SUCCESS ? "SUCCESS" : "FAIL");
return status;
@@ -680,6 +729,22 @@ namespace RcclUnitTesting
if (this->verbose) INFO("Child %d begins DestroyComms\n", this->childId);
// Release comms
for (int i = 0; i < this->comms.size(); ++i)
{
// Check if the communicator is non-blocking
if (this->useBlocking == false)
{
// handle the non-blocking case
ncclCommFinalize(this->comms[i]);
CHILD_NCCL_CALL_NON_BLOCKING("ncclCommGetAsyncErrorCommFinalize");
}
else
{
// In case of blocking just call Finalize
CHILD_NCCL_CALL(ncclCommFinalize(this->comms[i]), "ncclCommFinalize");
}
}
for (int i = 0; i < this->comms.size(); ++i)
{
CHILD_NCCL_CALL(ncclCommDestroy(this->comms[i]), "ncclCommDestroy");