From 0687940b84a8531172bdcc7563df184e23541886 Mon Sep 17 00:00:00 2001 From: gilbertlee-amd <44450918+gilbertlee-amd@users.noreply.github.com> Date: Mon, 7 Mar 2022 09:22:55 -0700 Subject: [PATCH] Changing initialization method for UnitTests (#510) --- test/AllReduce_GroupCall.cpp | 4 +- test/AllReduce_PreMultScalar.cpp | 4 +- test/SendRecv_SinglePairs.cpp | 4 +- test/common/TestBed.cpp | 63 +++++++++++++++++++++++--------- test/common/TestBedChild.cpp | 27 +++++++++++--- test/common/TestBedChild.hpp | 25 ++++++++----- 6 files changed, 88 insertions(+), 39 deletions(-) diff --git a/test/AllReduce_GroupCall.cpp b/test/AllReduce_GroupCall.cpp index 2b6d7ec77f..404935eb71 100644 --- a/test/AllReduce_GroupCall.cpp +++ b/test/AllReduce_GroupCall.cpp @@ -36,8 +36,8 @@ namespace RcclUnitTesting for (int dataIdx = 0; dataIdx < dataTypes.size() && isCorrect; ++dataIdx) { if (testBed.ev.showNames) - INFO("%s process %2d-ranks AllReduce %d Grouped Calls (%s-%s)\n", - isMultiProcess ? "Multi " : "Single", + INFO("%s %d-ranks AllReduce %d Grouped Calls (%s-%s)\n", + isMultiProcess ? "MP" : "SP", totalRanks, numCollPerGroup, ncclRedOpNames[redOps[redOpIdx]], ncclDataTypeNames[dataTypes[dataIdx]]); diff --git a/test/AllReduce_PreMultScalar.cpp b/test/AllReduce_PreMultScalar.cpp index ddf03c58a0..37d8a064b7 100644 --- a/test/AllReduce_PreMultScalar.cpp +++ b/test/AllReduce_PreMultScalar.cpp @@ -48,8 +48,8 @@ namespace RcclUnitTesting for (int scalarMode = 0; scalarMode <= 1 && isCorrect; ++scalarMode) { if (testBed.ev.showNames) - INFO("%s process %2d-ranks AllReduce (custom-scalar Mode %d %s)\n", - isMultiProcess ? "Multi " : "Single", + INFO("%s %d-ranks AllReduce (custom-scalar Mode %d %s)\n", + isMultiProcess ? "MP" : "SP", totalRanks, scalarMode, ncclDataTypeNames[dataType]); for (int i = 0; i < numElements.size() && isCorrect; ++i) diff --git a/test/SendRecv_SinglePairs.cpp b/test/SendRecv_SinglePairs.cpp index 857acbe1a9..e23a64f754 100644 --- a/test/SendRecv_SinglePairs.cpp +++ b/test/SendRecv_SinglePairs.cpp @@ -48,8 +48,8 @@ namespace RcclUnitTesting if (recvRank != sendRank) { if (testBed.ev.showNames) // Show test names - INFO("%s process Datatype: %s SendReceive test Rank %d -> Rank %d for %d Elements\n", - isMultiProcess ? "Multi " : "Single", + INFO("%s Datatype: %s SendReceive test Rank %d -> Rank %d for %d Elements\n", + isMultiProcess ? "MP" : "SP", ncclDataTypeNames[dataTypes[dataIdx]], sendRank, recvRank, diff --git a/test/common/TestBed.cpp b/test/common/TestBed.cpp index 7cebe46e83..6f0c2aba8f 100644 --- a/test/common/TestBed.cpp +++ b/test/common/TestBed.cpp @@ -10,11 +10,38 @@ #define PIPE_WRITE(childId, val) \ ASSERT_EQ(write(childList[childId]->parentWriteFd, &val, sizeof(val)), sizeof(val)) -#define PIPE_CHECK(childId) \ - { \ - int response = 0; \ - ASSERT_EQ(read(childList[childId]->parentReadFd, &response, sizeof(int)), sizeof(int)); \ - ASSERT_EQ(response, TEST_SUCCESS); \ + +#define PIPE_READ(childId, val) \ + { \ + if (ev.verbose) INFO("Calling PIPE_READ to Child %d\n", childId); \ + ssize_t retval = read(childList[childId]->parentReadFd, &val, sizeof(val)); \ + if (ev.verbose) INFO("Got PIPE_READ %ld\n", retval); \ + if (retval == -1) \ + { \ + ERROR("Unable to read from child %d: Error %s\n", childId, strerror(errno)); \ + FAIL(); \ + } \ + else if (retval == 0) \ + { \ + ERROR("Child %d pipe closed unexpectedly\n", childId); \ + exit(1); \ + } \ + else if (retval < sizeof(int)) \ + { \ + ERROR("Child %d pipe read incomplete (%ld / %lu)\n", childId, retval, sizeof(val)); \ + exit(1); \ + } \ + } + +#define PIPE_CHECK(childId) \ + { \ + int response = 0; \ + PIPE_READ(childId, response); \ + if (response != TEST_SUCCESS) \ + { \ + ERROR("Child %d reports failure\n", childId); \ + FAIL(); \ + } \ } namespace RcclUnitTesting @@ -24,19 +51,6 @@ namespace RcclUnitTesting numActiveChildren(0), numActiveRanks(0) { - // Set NCCL_COMM_ID to use a local port to avoid passing ncclCommId - // Calling ncclGetUniqueId would initialize HIP, which should not be done prior to fork - std::string localPort = "55513"; - if (!getenv("NCCL_COMM_ID")) - { - char hostname[HOST_NAME_MAX+1]; - gethostname(hostname, HOST_NAME_MAX+1); - std::string hostnameString(hostname); - hostnameString.append(":55513"); - setenv("NCCL_COMM_ID", hostnameString.c_str(), 0); - if (ev.verbose) INFO("NCCL_COMM_ID set to %s\n", hostnameString.c_str()); - } - // Collect the number of GPUs this->numDevicesAvailable = ev.maxGpus; if (ev.verbose) INFO("Detected %d GPUs\n", this->numDevicesAvailable); @@ -90,13 +104,26 @@ namespace RcclUnitTesting } } + // Tell first rank to get ncclUniqueId + int getIdCmd = TestBedChild::CHILD_GET_UNIQUE_ID; + PIPE_WRITE(0, getIdCmd); + + // Receive back unique ID from first rank + ncclUniqueId id; + PIPE_READ(0, id); + PIPE_CHECK(0); + // Send InitComms command to each active child process int const cmd = TestBedChild::CHILD_INIT_COMMS; int rankOffset = 0; for (int childId = 0; childId < this->numActiveChildren; ++childId) { + if (ev.verbose) INFO("Sending InitComm event to child %d\n", childId); PIPE_WRITE(childId, cmd); + // Send unique ID to child process + PIPE_WRITE(childId, id); + // Send total number of ranks to child process PIPE_WRITE(childId, this->numActiveRanks); diff --git a/test/common/TestBedChild.cpp b/test/common/TestBedChild.cpp index b936a5428e..a291f741c6 100644 --- a/test/common/TestBedChild.cpp +++ b/test/common/TestBedChild.cpp @@ -69,6 +69,7 @@ namespace RcclUnitTesting ErrCode status = TEST_SUCCESS; switch(command) { + case CHILD_GET_UNIQUE_ID : status = GetUniqueId(); break; case CHILD_INIT_COMMS : status = InitComms(); break; case CHILD_SET_COLL_ARGS : status = SetCollectiveArgs(); break; case CHILD_ALLOCATE_MEM : status = AllocateMem(); break; @@ -84,8 +85,13 @@ namespace RcclUnitTesting // Send back acknowledgement to parent if (status == TEST_FAIL) ERROR("Child %d failed on command [%s]:\n", this->childId, ChildCommandNames[command]); - write(childWriteFd, &status, sizeof(status)); + if (write(childWriteFd, &status, sizeof(status)) < 0) + { + ERROR("Child %d write to parent failed: %s\n", this->childId, strerror(errno)); + break; + } } + if (verbose) INFO("Child %d exiting execution loop\n", this->childId); // Close child ends of pipe close(this->childReadFd); @@ -94,11 +100,26 @@ namespace RcclUnitTesting exit(0); } + ErrCode TestBedChild::GetUniqueId() + { + if (this->verbose) INFO("Child %d begins GetUniqueId()\n", this->childId); + + // Get a unique ID and pass it back to parent process + ncclUniqueId id; + CHILD_NCCL_CALL(ncclGetUniqueId(&id), "ncclGetUniqueId"); + write(childWriteFd, &id, sizeof(id)); + + if (this->verbose) INFO("Child %d finishes GetUniqueId()\n", this->childId); + return TEST_SUCCESS; + } + ErrCode TestBedChild::InitComms() { if (this->verbose) INFO("Child %d begins InitComms()\n", this->childId); // Read values sent by parent [see TestBed::InitComms()] + ncclUniqueId id; + PIPE_READ(id); PIPE_READ(this->totalRanks); PIPE_READ(this->rankOffset); PIPE_READ(this->numCollectivesInGroup); @@ -116,10 +137,6 @@ namespace RcclUnitTesting this->collArgs[i].resize(numCollectivesInGroup); } - // Collect uniqueId (specified by NCCL_COMM_ID env var) - ncclUniqueId id; - CHILD_NCCL_CALL(ncclGetUniqueId(&id), "ncclGetUniqueId"); - // Initialize communicators comms.clear(); comms.resize(numGpus); diff --git a/test/common/TestBedChild.hpp b/test/common/TestBedChild.hpp index b6a0d011e5..e6ccd42444 100644 --- a/test/common/TestBedChild.hpp +++ b/test/common/TestBedChild.hpp @@ -20,20 +20,22 @@ namespace RcclUnitTesting // These are commands that can be given to the child process enum { - CHILD_INIT_COMMS = 0, // InitComms() - CHILD_SET_COLL_ARGS = 1, // SetCollectiveArgs() - CHILD_ALLOCATE_MEM = 2, // AllocateMem() - CHILD_PREPARE_DATA = 3, // PrepareData() - CHILD_EXECUTE_COLL = 4, // ExecuteCollectives() - CHILD_VALIDATE_RESULTS = 5, // ValidateResults() - CHILD_DEALLOCATE_MEM = 6, // DeallocateMem() - CHILD_DESTROY_COMMS = 7, // DestroyComms() - CHILD_STOP = 8, // Stop() - NUM_CHILD_COMMANDS = 9 + CHILD_GET_UNIQUE_ID = 0, // GetUniqueId() + CHILD_INIT_COMMS = 1, // InitComms() + CHILD_SET_COLL_ARGS = 2, // SetCollectiveArgs() + CHILD_ALLOCATE_MEM = 3, // AllocateMem() + CHILD_PREPARE_DATA = 4, // PrepareData() + CHILD_EXECUTE_COLL = 5, // ExecuteCollectives() + CHILD_VALIDATE_RESULTS = 6, // ValidateResults() + CHILD_DEALLOCATE_MEM = 7, // DeallocateMem() + CHILD_DESTROY_COMMS = 8, // DestroyComms() + CHILD_STOP = 9, // Stop() + NUM_CHILD_COMMANDS = 10 }; char const ChildCommandNames[NUM_CHILD_COMMANDS][20] = { + "GET_UNIQUE_ID", "INIT_COMMS", "SET_COLL_ARGS", "ALLOCATE_MEM", @@ -76,6 +78,9 @@ namespace RcclUnitTesting void StartExecutionLoop(); protected: + // Calls ncclGetUniqueId and returns it to parent + ErrCode GetUniqueId(); + // Initialize RCCL communicators ErrCode InitComms();