AI/rocm-systems
2
0
انشعاب 0
کد مسائل تقاضاهای واکشی Actions Packages پروژه‌ها انتشارها دانشنامه فعالیت

Changing initialization method for UnitTests (#510)

فهرست منبع
This commit is contained in:
gilbertlee-amd
2022-03-07 09:22:55 -07:00
کامیت شده توسط GitHub
والد d6d6af710e
کامیت 0687940b84
6فایلهای تغییر یافته به همراه88 افزوده شده و 39 حذف شده
دانلود پرونده وصله دانلود فایل تغییرات diff Expand all files Collapse all files
test/AllReduce_GroupCall.cpp
+2 -2
مشاهده پرونده
@@ -36,8 +36,8 @@ namespace RcclUnitTesting
for (int dataIdx = 0; dataIdx < dataTypes.size() && isCorrect; ++dataIdx)
{
if (testBed.ev.showNames)
INFO("%s process %2d-ranks AllReduce %d Grouped Calls (%s-%s)\n",
isMultiProcess ? "Multi " : "Single",
INFO("%s %d-ranks AllReduce %d Grouped Calls (%s-%s)\n",
isMultiProcess ? "MP" : "SP",
totalRanks, numCollPerGroup,
ncclRedOpNames[redOps[redOpIdx]], ncclDataTypeNames[dataTypes[dataIdx]]);
test/AllReduce_PreMultScalar.cpp
+2 -2
مشاهده پرونده
@@ -48,8 +48,8 @@ namespace RcclUnitTesting
for (int scalarMode = 0; scalarMode <= 1 && isCorrect; ++scalarMode)
{
if (testBed.ev.showNames)
INFO("%s process %2d-ranks AllReduce (custom-scalar Mode %d %s)\n",
isMultiProcess ? "Multi " : "Single",
INFO("%s %d-ranks AllReduce (custom-scalar Mode %d %s)\n",
isMultiProcess ? "MP" : "SP",
totalRanks, scalarMode, ncclDataTypeNames[dataType]);
for (int i = 0; i < numElements.size() && isCorrect; ++i)
test/SendRecv_SinglePairs.cpp
+2 -2
مشاهده پرونده
@@ -48,8 +48,8 @@ namespace RcclUnitTesting
if (recvRank != sendRank)
{
if (testBed.ev.showNames) // Show test names
INFO("%s process Datatype: %s SendReceive test Rank %d -> Rank %d for %d Elements\n",
isMultiProcess ? "Multi " : "Single",
INFO("%s Datatype: %s SendReceive test Rank %d -> Rank %d for %d Elements\n",
isMultiProcess ? "MP" : "SP",
ncclDataTypeNames[dataTypes[dataIdx]],
sendRank,
recvRank,
test/common/TestBed.cpp
+45 -18
مشاهده پرونده
@@ -10,11 +10,38 @@
#define PIPE_WRITE(childId, val) \
ASSERT_EQ(write(childList[childId]->parentWriteFd, &val, sizeof(val)), sizeof(val))
#define PIPE_CHECK(childId) \
{ \
int response = 0; \
ASSERT_EQ(read(childList[childId]->parentReadFd, &response, sizeof(int)), sizeof(int)); \
ASSERT_EQ(response, TEST_SUCCESS); \
#define PIPE_READ(childId, val) \
{ \
if (ev.verbose) INFO("Calling PIPE_READ to Child %d\n", childId); \
ssize_t retval = read(childList[childId]->parentReadFd, &val, sizeof(val)); \
if (ev.verbose) INFO("Got PIPE_READ %ld\n", retval); \
if (retval == -1) \
{ \
ERROR("Unable to read from child %d: Error %s\n", childId, strerror(errno)); \
FAIL(); \
} \
else if (retval == 0) \
{ \
ERROR("Child %d pipe closed unexpectedly\n", childId); \
exit(1); \
} \
else if (retval < sizeof(int)) \
{ \
ERROR("Child %d pipe read incomplete (%ld / %lu)\n", childId, retval, sizeof(val)); \
exit(1); \
} \
}
#define PIPE_CHECK(childId) \
{ \
int response = 0; \
PIPE_READ(childId, response); \
if (response != TEST_SUCCESS) \
{ \
ERROR("Child %d reports failure\n", childId); \
FAIL(); \
} \
}
namespace RcclUnitTesting
@@ -24,19 +51,6 @@ namespace RcclUnitTesting
numActiveChildren(0),
numActiveRanks(0)
{
// Set NCCL_COMM_ID to use a local port to avoid passing ncclCommId
// Calling ncclGetUniqueId would initialize HIP, which should not be done prior to fork
std::string localPort = "55513";
if (!getenv("NCCL_COMM_ID"))
{
char hostname[HOST_NAME_MAX+1];
gethostname(hostname, HOST_NAME_MAX+1);
std::string hostnameString(hostname);
hostnameString.append(":55513");
setenv("NCCL_COMM_ID", hostnameString.c_str(), 0);
if (ev.verbose) INFO("NCCL_COMM_ID set to %s\n", hostnameString.c_str());
}
// Collect the number of GPUs
this->numDevicesAvailable = ev.maxGpus;
if (ev.verbose) INFO("Detected %d GPUs\n", this->numDevicesAvailable);
@@ -90,13 +104,26 @@ namespace RcclUnitTesting
}
}
// Tell first rank to get ncclUniqueId
int getIdCmd = TestBedChild::CHILD_GET_UNIQUE_ID;
PIPE_WRITE(0, getIdCmd);
// Receive back unique ID from first rank
ncclUniqueId id;
PIPE_READ(0, id);
PIPE_CHECK(0);
// Send InitComms command to each active child process
int const cmd = TestBedChild::CHILD_INIT_COMMS;
int rankOffset = 0;
for (int childId = 0; childId < this->numActiveChildren; ++childId)
{
if (ev.verbose) INFO("Sending InitComm event to child %d\n", childId);
PIPE_WRITE(childId, cmd);
// Send unique ID to child process
PIPE_WRITE(childId, id);
// Send total number of ranks to child process
PIPE_WRITE(childId, this->numActiveRanks);
test/common/TestBedChild.cpp
+22 -5
مشاهده پرونده
@@ -69,6 +69,7 @@ namespace RcclUnitTesting
ErrCode status = TEST_SUCCESS;
switch(command)
{
case CHILD_GET_UNIQUE_ID : status = GetUniqueId(); break;
case CHILD_INIT_COMMS : status = InitComms(); break;
case CHILD_SET_COLL_ARGS : status = SetCollectiveArgs(); break;
case CHILD_ALLOCATE_MEM : status = AllocateMem(); break;
@@ -84,8 +85,13 @@ namespace RcclUnitTesting
// Send back acknowledgement to parent
if (status == TEST_FAIL)
ERROR("Child %d failed on command [%s]:\n", this->childId, ChildCommandNames[command]);
write(childWriteFd, &status, sizeof(status));
if (write(childWriteFd, &status, sizeof(status)) < 0)
{
ERROR("Child %d write to parent failed: %s\n", this->childId, strerror(errno));
break;
}
}
if (verbose) INFO("Child %d exiting execution loop\n", this->childId);
// Close child ends of pipe
close(this->childReadFd);
@@ -94,11 +100,26 @@ namespace RcclUnitTesting
exit(0);
}
ErrCode TestBedChild::GetUniqueId()
{
if (this->verbose) INFO("Child %d begins GetUniqueId()\n", this->childId);
// Get a unique ID and pass it back to parent process
ncclUniqueId id;
CHILD_NCCL_CALL(ncclGetUniqueId(&id), "ncclGetUniqueId");
write(childWriteFd, &id, sizeof(id));
if (this->verbose) INFO("Child %d finishes GetUniqueId()\n", this->childId);
return TEST_SUCCESS;
}
ErrCode TestBedChild::InitComms()
{
if (this->verbose) INFO("Child %d begins InitComms()\n", this->childId);
// Read values sent by parent [see TestBed::InitComms()]
ncclUniqueId id;
PIPE_READ(id);
PIPE_READ(this->totalRanks);
PIPE_READ(this->rankOffset);
PIPE_READ(this->numCollectivesInGroup);
@@ -116,10 +137,6 @@ namespace RcclUnitTesting
this->collArgs[i].resize(numCollectivesInGroup);
}
// Collect uniqueId (specified by NCCL_COMM_ID env var)
ncclUniqueId id;
CHILD_NCCL_CALL(ncclGetUniqueId(&id), "ncclGetUniqueId");
// Initialize communicators
comms.clear();
comms.resize(numGpus);
test/common/TestBedChild.hpp
+15 -10
مشاهده پرونده
@@ -20,20 +20,22 @@ namespace RcclUnitTesting
// These are commands that can be given to the child process
enum
{
CHILD_INIT_COMMS = 0, // InitComms()
CHILD_SET_COLL_ARGS = 1, // SetCollectiveArgs()
CHILD_ALLOCATE_MEM = 2, // AllocateMem()
CHILD_PREPARE_DATA = 3, // PrepareData()
CHILD_EXECUTE_COLL = 4, // ExecuteCollectives()
CHILD_VALIDATE_RESULTS = 5, // ValidateResults()
CHILD_DEALLOCATE_MEM = 6, // DeallocateMem()
CHILD_DESTROY_COMMS = 7, // DestroyComms()
CHILD_STOP = 8, // Stop()
NUM_CHILD_COMMANDS = 9
CHILD_GET_UNIQUE_ID = 0, // GetUniqueId()
CHILD_INIT_COMMS = 1, // InitComms()
CHILD_SET_COLL_ARGS = 2, // SetCollectiveArgs()
CHILD_ALLOCATE_MEM = 3, // AllocateMem()
CHILD_PREPARE_DATA = 4, // PrepareData()
CHILD_EXECUTE_COLL = 5, // ExecuteCollectives()
CHILD_VALIDATE_RESULTS = 6, // ValidateResults()
CHILD_DEALLOCATE_MEM = 7, // DeallocateMem()
CHILD_DESTROY_COMMS = 8, // DestroyComms()
CHILD_STOP = 9, // Stop()
NUM_CHILD_COMMANDS = 10
};
char const ChildCommandNames[NUM_CHILD_COMMANDS][20] =
{
"GET_UNIQUE_ID",
"INIT_COMMS",
"SET_COLL_ARGS",
"ALLOCATE_MEM",
@@ -76,6 +78,9 @@ namespace RcclUnitTesting
void StartExecutionLoop();
protected:
// Calls ncclGetUniqueId and returns it to parent
ErrCode GetUniqueId();
// Initialize RCCL communicators
ErrCode InitComms();