Merge commit '3d4813d99196bb349eccd50a925e2addc8f1622c' into develop
This commit is contained in:
@@ -192,8 +192,8 @@ namespace RcclUnitTesting
|
||||
case ncclCollAllReduce: ss << "ncclAllReduce"; break;
|
||||
case ncclCollGather: ss << "ncclGather"; break;
|
||||
case ncclCollScatter: ss << "ncclScatter"; break;
|
||||
case ncclCollAllToAll: ss << "ncclAllToAll"; break;
|
||||
case ncclCollAllToAllv: ss << "ncclAllToAllv"; break;
|
||||
case ncclCollAlltoAll: ss << "ncclAlltoAll"; break;
|
||||
case ncclCollAlltoAllv: ss << "ncclAlltoAllv"; break;
|
||||
case ncclCollSend: ss << "ncclSend"; break;
|
||||
case ncclCollRecv: ss << "ncclRecv"; break;
|
||||
default: ss << "[Unknown]"; break;
|
||||
@@ -276,7 +276,7 @@ namespace RcclUnitTesting
|
||||
*numInputElements = totalRanks * N;
|
||||
*numOutputElements = N;
|
||||
break;
|
||||
case ncclCollAllToAll:
|
||||
case ncclCollAlltoAll:
|
||||
*numInputElements = totalRanks * N;
|
||||
*numOutputElements = totalRanks * N;
|
||||
break;
|
||||
|
||||
@@ -20,8 +20,8 @@ namespace RcclUnitTesting
|
||||
ncclCollAllReduce,
|
||||
ncclCollGather,
|
||||
ncclCollScatter,
|
||||
ncclCollAllToAll,
|
||||
ncclCollAllToAllv,
|
||||
ncclCollAlltoAll,
|
||||
ncclCollAlltoAllv,
|
||||
ncclCollSend,
|
||||
ncclCollRecv,
|
||||
ncclNumFuncs
|
||||
@@ -36,8 +36,8 @@ namespace RcclUnitTesting
|
||||
"AllReduce",
|
||||
"Gather",
|
||||
"Scatter",
|
||||
"AllToAll",
|
||||
"AllToAllv",
|
||||
"AlltoAll",
|
||||
"AlltoAllv",
|
||||
"Send",
|
||||
"Recv"
|
||||
};
|
||||
|
||||
@@ -22,8 +22,8 @@ namespace RcclUnitTesting
|
||||
case ncclCollAllReduce: return DefaultPrepData_Reduce(collArgs, true);
|
||||
case ncclCollGather: return DefaultPrepData_Gather(collArgs, false);
|
||||
case ncclCollScatter: return DefaultPrepData_Scatter(collArgs);
|
||||
case ncclCollAllToAll: return DefaultPrepData_AllToAll(collArgs);
|
||||
case ncclCollAllToAllv: return DefaultPrepData_AllToAllv(collArgs);
|
||||
case ncclCollAlltoAll: return DefaultPrepData_AllToAll(collArgs);
|
||||
case ncclCollAlltoAllv: return DefaultPrepData_AllToAllv(collArgs);
|
||||
case ncclCollSend: return DefaultPrepData_Send(collArgs);
|
||||
case ncclCollRecv: return DefaultPrepData_Recv(collArgs);
|
||||
default:
|
||||
|
||||
@@ -402,6 +402,9 @@ bool ProcessIsolatedTestRunner::executeAllTests(const ExecutionOptions& options)
|
||||
continue;
|
||||
}
|
||||
|
||||
// Flush all output before fork to prevent child from inheriting unflushed buffers
|
||||
fflush(NULL);
|
||||
|
||||
pid_t pid = fork();
|
||||
|
||||
if(pid == 0)
|
||||
@@ -435,6 +438,10 @@ bool ProcessIsolatedTestRunner::executeAllTests(const ExecutionOptions& options)
|
||||
{
|
||||
INFO("Running isolated test '%s' (PID: %d)\n", testConfig.name.c_str(), pid);
|
||||
}
|
||||
// Flush parent's output before reading from child pipes to ensure proper ordering
|
||||
fflush(stdout);
|
||||
fflush(stderr);
|
||||
|
||||
int status;
|
||||
CapturedOutput output = captureProcessOutput(stdout_fd, stderr_fd, pid, &status);
|
||||
|
||||
@@ -442,6 +449,9 @@ bool ProcessIsolatedTestRunner::executeAllTests(const ExecutionOptions& options)
|
||||
auto duration
|
||||
= std::chrono::duration_cast<std::chrono::milliseconds>(endTime - startTime);
|
||||
|
||||
// Display captured output BEFORE status messages for proper sequencing
|
||||
displayCapturedOutput(output, testConfig.name);
|
||||
|
||||
TestResult testResult;
|
||||
testResult.testName = testConfig.name;
|
||||
testResult.processId = pid;
|
||||
@@ -530,8 +540,6 @@ bool ProcessIsolatedTestRunner::executeAllTests(const ExecutionOptions& options)
|
||||
testResult.errorMessage = "Failed to wait for process";
|
||||
}
|
||||
|
||||
displayCapturedOutput(output, testConfig.name);
|
||||
|
||||
recordTestResult(testResult);
|
||||
|
||||
// Stop on first failure if requested
|
||||
|
||||
@@ -647,18 +647,18 @@ namespace RcclUnitTesting
|
||||
this->streams[groupId][localRank][collArg.streamIdx]),
|
||||
"ncclScatter");
|
||||
break;
|
||||
case ncclCollAllToAll:
|
||||
CHILD_NCCL_CALL_RANK(errCode, ncclAllToAll(
|
||||
case ncclCollAlltoAll:
|
||||
CHILD_NCCL_CALL_RANK(errCode, ncclAlltoAll(
|
||||
collArg.inputGpu.ptr,
|
||||
collArg.outputGpu.ptr,
|
||||
collArg.numInputElements / collArg.totalRanks,
|
||||
collArg.dataType,
|
||||
this->comms[localRank],
|
||||
this->streams[groupId][localRank][collArg.streamIdx]),
|
||||
"ncclAllToAll");
|
||||
"ncclAlltoAll");
|
||||
break;
|
||||
case ncclCollAllToAllv:
|
||||
CHILD_NCCL_CALL_RANK(errCode, ncclAllToAllv(
|
||||
case ncclCollAlltoAllv:
|
||||
CHILD_NCCL_CALL_RANK(errCode, ncclAlltoAllv(
|
||||
collArg.inputGpu.ptr,
|
||||
collArg.options.sendcounts + (this->rankOffset + localRank)*this->totalRanks,
|
||||
collArg.options.sdispls + (this->rankOffset + localRank)*this->totalRanks,
|
||||
@@ -668,7 +668,7 @@ namespace RcclUnitTesting
|
||||
collArg.dataType,
|
||||
this->comms[localRank],
|
||||
this->streams[groupId][localRank][collArg.streamIdx]),
|
||||
"ncclAllToAllv");
|
||||
"ncclAlltoAllv");
|
||||
break;
|
||||
case ncclCollSend:
|
||||
CHILD_NCCL_CALL_RANK(errCode, ncclSend(
|
||||
|
||||
Reference in New Issue
Block a user