replayer update v0 (#1733)
* First version of new replayer, with comments on future TODOs * plus minor fixes for UT * Updated format of recorder, especially in binary department, according to replayer's need
This commit is contained in:
Tim
GitHub
@@ -170,12 +170,14 @@ ncclResult_t ncclAllToAll_impl(const void* sendbuff, void* recvbuff, size_t coun
|
||||
int nRanks;
|
||||
NCCLCHECK(ncclCommCount(comm, &nRanks));
|
||||
if (count == 0) return ncclSuccess;
|
||||
if (!mscclIsCaller()) Recorder::instance().skip(true);
|
||||
NCCLCHECK(ncclGroupStart());
|
||||
for (int r=0; r<nRanks; r++) {
|
||||
NCCLCHECK(ncclSend(((char*)sendbuff)+r*rankOffset, count, datatype, r, comm, stream));
|
||||
NCCLCHECK(ncclRecv(((char*)recvbuff)+r*rankOffset, count, datatype, r, comm, stream));
|
||||
}
|
||||
NCCLCHECK(ncclGroupEnd());
|
||||
if (!mscclIsCaller()) Recorder::instance().skip(false);
|
||||
return ncclSuccess;
|
||||
}
|
||||
}
|
||||
@@ -205,6 +207,7 @@ ncclResult_t ncclAllToAllv_impl(const void *sendbuff, const size_t sendcounts[],
|
||||
|
||||
int nRanks;
|
||||
NCCLCHECK(ncclCommCount(comm, &nRanks));
|
||||
if (!mscclIsCaller()) Recorder::instance().skip(true);
|
||||
NCCLCHECK(ncclGroupStart());
|
||||
for (int r=0; r<nRanks; r++) {
|
||||
NCCLCHECK(ncclSend(
|
||||
@@ -223,6 +226,7 @@ ncclResult_t ncclAllToAllv_impl(const void *sendbuff, const size_t sendcounts[],
|
||||
stream));
|
||||
}
|
||||
NCCLCHECK(ncclGroupEnd());
|
||||
if (!mscclIsCaller()) Recorder::instance().skip(false);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -285,6 +289,7 @@ ncclResult_t ncclGather_impl(const void* sendbuff, void* recvbuff, size_t sendco
|
||||
if (sendcount == 0) return ncclSuccess;
|
||||
int rank;
|
||||
NCCLCHECK(ncclCommUserRank(comm, &rank));
|
||||
if (!mscclIsCaller()) Recorder::instance().skip(true);
|
||||
NCCLCHECK(ncclGroupStart());
|
||||
if (rank == root) {
|
||||
for (int r=0; r<nRanks; r++)
|
||||
@@ -292,6 +297,7 @@ ncclResult_t ncclGather_impl(const void* sendbuff, void* recvbuff, size_t sendco
|
||||
}
|
||||
NCCLCHECK(ncclSend(sendbuff, sendcount, datatype, root, comm, stream));
|
||||
NCCLCHECK(ncclGroupEnd());
|
||||
if (!mscclIsCaller()) Recorder::instance().skip(false);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
@@ -374,6 +380,7 @@ ncclResult_t ncclScatter_impl(const void* sendbuff, void* recvbuff, size_t recvc
|
||||
if (recvcount == 0) return ncclSuccess;
|
||||
int rank;
|
||||
NCCLCHECK(ncclCommUserRank(comm, &rank));
|
||||
if (!mscclIsCaller()) Recorder::instance().skip(true);
|
||||
NCCLCHECK(ncclGroupStart());
|
||||
if (rank == root) {
|
||||
for (int r=0; r<nRanks; r++)
|
||||
@@ -381,6 +388,7 @@ ncclResult_t ncclScatter_impl(const void* sendbuff, void* recvbuff, size_t recvc
|
||||
}
|
||||
NCCLCHECK(ncclRecv(recvbuff, recvcount, datatype, root, comm, stream));
|
||||
NCCLCHECK(ncclGroupEnd());
|
||||
if (!mscclIsCaller()) Recorder::instance().skip(false);
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
|
||||
+12
-3
@@ -98,7 +98,10 @@ ncclResult_t ncclAsyncJobComplete(struct ncclAsyncJob* job) {
|
||||
|
||||
NCCL_API(ncclResult_t, ncclGroupStart);
|
||||
ncclResult_t ncclGroupStart_impl() {
|
||||
NCCLCHECK(Recorder::instance().record(rrGroupStart, ncclGroupDepth));
|
||||
if (!mscclIsCaller())
|
||||
{
|
||||
NCCLCHECK(Recorder::instance().record(rrGroupStart, ncclGroupDepth));
|
||||
}
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
NVTX3_FUNC_RANGE_IN(nccl_domain);
|
||||
|
||||
@@ -117,7 +120,10 @@ ncclResult_t ncclGroupStartInternal() {
|
||||
|
||||
NCCL_API(ncclResult_t, ncclGroupEnd);
|
||||
ncclResult_t ncclGroupEnd_impl() {
|
||||
NCCLCHECK(Recorder::instance().record(rrGroupEnd, ncclGroupDepth));
|
||||
if (!mscclIsCaller())
|
||||
{
|
||||
NCCLCHECK(Recorder::instance().record(rrGroupEnd, ncclGroupDepth));
|
||||
}
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
NVTX3_FUNC_RANGE_IN(nccl_domain);
|
||||
NCCLCHECKGOTO(ncclGroupEndInternal(), ret, exit);
|
||||
@@ -128,7 +134,10 @@ exit:
|
||||
|
||||
NCCL_API(ncclResult_t, ncclGroupSimulateEnd, ncclSimInfo_t* simInfo);
|
||||
ncclResult_t ncclGroupSimulateEnd(ncclSimInfo_t* simInfo) {
|
||||
Recorder::instance().record(ncclGroupDepth, simInfo);
|
||||
if (!mscclIsCaller())
|
||||
{
|
||||
Recorder::instance().record(ncclGroupDepth, simInfo);
|
||||
}
|
||||
ncclResult_t ret = ncclSuccess;
|
||||
NVTX3_FUNC_RANGE_IN(nccl_domain);
|
||||
NCCLCHECKGOTO(ncclGroupEndInternal(simInfo), ret, exit);
|
||||
|
||||
+11
-5
@@ -90,17 +90,21 @@ struct rcclApiCall {
|
||||
int graphCaptured = -1;
|
||||
|
||||
// explicit data from header
|
||||
rcclCall_t type; // in adjacent to op Name ^
|
||||
rcclCall_t type;
|
||||
uint64_t opCount = 0;
|
||||
const void* sendbuff = NULL;
|
||||
void* recvbuff = NULL;
|
||||
void* sendPtrBase = NULL;
|
||||
void* recvPtrBase = NULL;
|
||||
size_t sendPtrExtent = 0;
|
||||
size_t recvPtrExtent = 0;
|
||||
size_t count = 0;
|
||||
ncclDataType_t datatype;
|
||||
ncclRedOp_t op;
|
||||
int root = -1;
|
||||
int nRanks = -1;
|
||||
ncclComm_t comm;
|
||||
hipStream_t stream;
|
||||
ncclComm_t comm = NULL;
|
||||
hipStream_t stream = NULL;
|
||||
int nTasks = -1;
|
||||
int globalRank = -1;
|
||||
uint64_t commId = 0;
|
||||
@@ -112,14 +116,15 @@ struct rcclApiCall {
|
||||
|
||||
class Recorder {
|
||||
private:
|
||||
std::ofstream outputFile; //1 per process
|
||||
std::ofstream outputFile; // 1 per process
|
||||
int output_json = 0; // 0 is to binary, 1 to json
|
||||
std::string filename;
|
||||
int logLevel = -1;
|
||||
|
||||
//std::string hostname;
|
||||
int pid = -1;
|
||||
int numCall = 0; //for debugging only
|
||||
int numCall = 0; // reserved for future record format/debug
|
||||
bool skipped = false; // number of sendrecv calls to skip for gather/scatter/a2a(v)
|
||||
static __thread int rcclReplayThreadIdx;
|
||||
static int depth; // for indentation purpose, will need thread safty later
|
||||
|
||||
@@ -136,6 +141,7 @@ class Recorder {
|
||||
|
||||
public:
|
||||
static Recorder& instance();
|
||||
void skip(bool b);
|
||||
void record(const char* name); // non-replayable calls
|
||||
ncclResult_t record(rcclApiCall& call);
|
||||
ncclResult_t record(rcclCall_t type, const ncclInfo& info); // collective
|
||||
|
||||
+53
-25
@@ -43,14 +43,21 @@ rcclApiCall::rcclApiCall(rcclCall_t type, const ncclInfo& info)://name(rcclCallS
|
||||
nRanks(info.comm->nRanks),
|
||||
stream(info.stream),
|
||||
nTasks(info.comm->planner.nTasksP2p + info.comm->planner.nTasksColl),
|
||||
globalRank(info.comm->localRankToRank[info.comm->localRank]){}
|
||||
globalRank(info.comm->localRankToRank[info.comm->localRank])
|
||||
{
|
||||
hipMemGetAddressRange(&recvPtrBase, &recvPtrExtent, const_cast<void*>(info.recvbuff)); // should always exist for collectives
|
||||
if (info.sendbuff) // ncclSend/Recv
|
||||
{
|
||||
hipMemGetAddressRange(&sendPtrBase, &sendPtrExtent, const_cast<void*>(info.sendbuff));
|
||||
}
|
||||
}
|
||||
|
||||
rcclApiCall::rcclApiCall(rcclCall_t type) : type(type){}
|
||||
|
||||
std::string siminfo_fmt = "[size : %zu, magic : %u, version : %u, estimated time : %f, timestamp : %f]";
|
||||
std::string config_fmt = ", ncclConfig : [size : %zu, magic : %u, version : %u, blocking : %d, cgaClusterSize : %d, minCTA : %d, maxCTA : %d, netname : %s, splitshare : %d]";
|
||||
std::string ctxt_fmt = "time : %lf, thread : %d, device : %d, captured : %d, graphID : %llu ]]"; // implicit context info
|
||||
std::string ubr_fmt = "%s : [comm : %p, buff : %p, returned handle : %p, size : %zu, context : [";
|
||||
std::string ubr_fmt = "%s : [comm : %p, buff : [addr : %p, base : %p, size : %zu], returned handle : %p, count : %zu, context : [";
|
||||
std::string getId_fmt = "%s : [uniqueID : %llu, context : [";
|
||||
std::string ubDereg_fmt = "%s : [comm : %p, handle : %p, context : [";
|
||||
std::string rank_fmt = "%s : [size : %d, uniqueID : %llu, rank : %d, context : [";
|
||||
@@ -62,7 +69,7 @@ std::string alloc_fmt = "%s : [returned ptr : %p, size : %zu, context : [";
|
||||
std::string free_fmt = "%s : [ptr : %p, context : [";
|
||||
std::string redop_fmt = "%s : [scalar : %p, datatype : %d, op : %d, residence : %d, comm : %p, context : [";
|
||||
std::string redopdestroy_fmt = "%s : [op : %d, comm : %p, context : [";
|
||||
std::string coll_fmt = "%s : [opCount : %lx, sendbuff : %p, recvbuff : %p, count : %zu, datatype : %d, op : %d, root : %d, comm : %p, nranks : %d, stream : %p, task : %d, globalrank : %d, context : [";
|
||||
std::string coll_fmt = "%s : [opCount : %lx, sendbuff : [addr : %p, base : %p, size : %zu], recvbuff : [addr : %p, base : %p, size : %zu], count : %zu, datatype : %d, op : %d, root : %d, comm : %p, nranks : %d, stream : %p, task : %d, globalrank : %d, context : [";
|
||||
|
||||
Recorder::Recorder()
|
||||
{
|
||||
@@ -74,8 +81,8 @@ Recorder::Recorder()
|
||||
}
|
||||
|
||||
logLevel = getenv("RCCL_LOG_LEVEL") ? std::stoi(getenv("RCCL_LOG_LEVEL")) : 1;
|
||||
char hostname[1024];
|
||||
getHostName(hostname, 1024, '.');
|
||||
char hostname[256];
|
||||
gethostname(hostname, 256);
|
||||
pid = getpid();
|
||||
output_json = 0;
|
||||
|
||||
@@ -94,13 +101,13 @@ Recorder::Recorder()
|
||||
output_name = std::string(filename);
|
||||
}
|
||||
|
||||
outputFile.open(output_name + std::to_string(pid) + output_extension,
|
||||
outputFile.open(output_name + "." + std::to_string(pid) + "." + std::string(hostname) + output_extension,
|
||||
output_json ? std::ofstream::out : std::ofstream::binary);
|
||||
if (output_json)
|
||||
{
|
||||
outputFile << "{" << std::endl;
|
||||
indent(2, outputFile);
|
||||
outputFile << "hostname : " << hostname << ", version : 0,";
|
||||
outputFile << "version : 1,";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,6 +117,14 @@ Recorder& Recorder::instance()
|
||||
return _instance;
|
||||
}
|
||||
|
||||
void Recorder::skip(bool b)
|
||||
{
|
||||
if (filename.size())
|
||||
{
|
||||
skipped = b;
|
||||
}
|
||||
}
|
||||
|
||||
void Recorder::captureGpuContext(rcclApiCall& call) const
|
||||
{
|
||||
call.timestamp = duration_cast<duration<double>>(high_resolution_clock::now().time_since_epoch()).count() * 1000;
|
||||
@@ -171,7 +186,7 @@ void Recorder::write(const rcclApiCall &call)
|
||||
case rrCommRegister:
|
||||
{
|
||||
len = snprintf(buffer, 4096, ubr_fmt.c_str(),
|
||||
rcclCallStr[call.type], call.comm, call.sendbuff, call.recvbuff, call.count);
|
||||
rcclCallStr[call.type], call.comm, call.sendbuff, call.sendPtrBase, call.sendPtrExtent, call.recvbuff, call.count);
|
||||
break;
|
||||
}
|
||||
case rrCommDeregister:
|
||||
@@ -240,7 +255,8 @@ void Recorder::write(const rcclApiCall &call)
|
||||
}
|
||||
default: // collectives
|
||||
len = snprintf(buffer, 4096, coll_fmt.c_str(),
|
||||
rcclCallStr[call.type], call.opCount, call.sendbuff, call.recvbuff, call.count, call.datatype,
|
||||
rcclCallStr[call.type], call.opCount, call.sendbuff, call.sendPtrBase, call.sendPtrExtent,
|
||||
call.recvbuff, call.recvPtrBase, call.recvPtrExtent, call.count, call.datatype,
|
||||
call.op, call.root, call.comm, call.nRanks, call.stream, call.nTasks, call.globalRank);
|
||||
|
||||
}
|
||||
@@ -249,7 +265,6 @@ void Recorder::write(const rcclApiCall &call)
|
||||
outputFile.write(buffer, len);
|
||||
} else {
|
||||
outputFile.write((char*)&call, sizeof(rcclApiCall));
|
||||
outputFile << std::endl;
|
||||
}
|
||||
outputFile.flush();
|
||||
return ;
|
||||
@@ -314,7 +329,6 @@ ncclResult_t Recorder::record(rcclApiCall& call)
|
||||
#if ROCM_VERSION >= 60100
|
||||
hipStreamCaptureStatus status;
|
||||
hipGraph_t graphCaptured;
|
||||
unsigned long long graphID = 0;
|
||||
CUDACHECK(hipStreamGetCaptureInfo_v2(call.stream, &status, &(call.graphID), &graphCaptured)); // shouldnt we need dependency?
|
||||
|
||||
if (status == hipStreamCaptureStatusActive) // when graph launched this should be disabled
|
||||
@@ -338,7 +352,7 @@ ncclResult_t Recorder::record(rcclApiCall& call)
|
||||
|
||||
ncclResult_t Recorder::record(rcclCall_t type, const ncclInfo& info)
|
||||
{
|
||||
if (!filename.size())
|
||||
if (!filename.size() || skipped)
|
||||
{
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -366,9 +380,9 @@ ncclResult_t Recorder::record(rcclCall_t type, const void* sendbuff, void* recvb
|
||||
ncclResult_t ret = record(call);
|
||||
if (type == rrAllToAllv)
|
||||
{
|
||||
int size = call.nRanks - 1;
|
||||
if (output_json)
|
||||
{
|
||||
int size = call.nRanks - 1;
|
||||
outputFile << ", sendcounts : [";
|
||||
for (int i = 0; i < size; i++) outputFile << sendcounts[i] << ", ";
|
||||
outputFile << sendcounts[size] << "], sdispls : [";
|
||||
@@ -378,9 +392,13 @@ ncclResult_t Recorder::record(rcclCall_t type, const void* sendbuff, void* recvb
|
||||
outputFile << recvcounts[size] << "], rdispls : [";
|
||||
for (int i = 0; i < size; i++) outputFile << rdispls[i] << ", ";
|
||||
outputFile << rdispls[size] << "]";
|
||||
outputFile.flush();
|
||||
} else {
|
||||
outputFile.write((char*)sendcounts, sizeof(size_t) * (size + 1));
|
||||
outputFile.write((char*)sdispls, sizeof(size_t) * (size + 1));
|
||||
outputFile.write((char*)recvcounts, sizeof(size_t) * (size + 1));
|
||||
outputFile.write((char*)rdispls, sizeof(size_t) * (size + 1));
|
||||
}
|
||||
// else export to binary
|
||||
outputFile.flush();
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
@@ -406,7 +424,7 @@ ncclResult_t Recorder::record(rcclCall_t type, ncclRedOp_t op, ncclComm_t comm,
|
||||
|
||||
ncclResult_t Recorder::record(rcclCall_t type, int groupDepth)
|
||||
{
|
||||
if (!filename.size())
|
||||
if (!filename.size() || skipped)
|
||||
{
|
||||
return ncclSuccess;
|
||||
}
|
||||
@@ -467,6 +485,7 @@ ncclResult_t Recorder::record(rcclCall_t type, ncclComm_t comm, void* handle, vo
|
||||
call.recvbuff = handle;
|
||||
if (type == rrCommRegister)
|
||||
{
|
||||
CUDACHECK(hipMemGetAddressRange(&call.sendPtrBase, &call.sendPtrExtent, userBuffer));
|
||||
call.sendbuff = userBuffer;
|
||||
call.count = size;
|
||||
}
|
||||
@@ -503,6 +522,7 @@ void Recorder::record(int groupDepth, ncclSimInfo_t *siminfo)
|
||||
siminfo->size, siminfo->magic, siminfo->version, siminfo->estimatedTime, call.timestamp);
|
||||
outputFile.write(buffer, len);
|
||||
} // no tid for groupCall
|
||||
// TODO: else flush siminfo in binary
|
||||
outputFile.flush();
|
||||
}
|
||||
|
||||
@@ -529,6 +549,7 @@ void Recorder::record(rcclCall_t type, int size, int rank, ncclUniqueId* commId,
|
||||
outputFile.write(buffer, len);
|
||||
outputFile.flush();
|
||||
}
|
||||
// TODO: else flush ncclConfig in binary
|
||||
}
|
||||
|
||||
void Recorder::record(ncclComm_t* comms, int ndev, const int* devlist)
|
||||
@@ -540,15 +561,20 @@ void Recorder::record(ncclComm_t* comms, int ndev, const int* devlist)
|
||||
|
||||
rcclApiCall call(rrCommInitAll);
|
||||
call.root = ndev;
|
||||
// call.sendbuff = comms; TODO: might log this too
|
||||
call.sendbuff = devlist;
|
||||
record(call);
|
||||
|
||||
if (output_json && devlist)
|
||||
if (devlist)
|
||||
{
|
||||
outputFile << ", devlist : [";
|
||||
for (int i = 0; i < call.root - 1; i++)
|
||||
outputFile << devlist[i] << ", ";
|
||||
outputFile << devlist[call.root - 1] << "]";
|
||||
if (output_json)
|
||||
{
|
||||
outputFile << ", devlist : [";
|
||||
for (int i = 0; i < call.root - 1; i++)
|
||||
outputFile << devlist[i] << ", ";
|
||||
outputFile << devlist[call.root - 1] << "]";
|
||||
} else {
|
||||
outputFile.write((char*)devlist, sizeof(int) * ndev);
|
||||
}
|
||||
outputFile.flush();
|
||||
}
|
||||
}
|
||||
@@ -578,6 +604,7 @@ static rcclCall_t getFuncType(std::string func)
|
||||
|
||||
void parseJsonEntry(const char* entry, std::vector<rcclApiCall>& calls)
|
||||
{
|
||||
// TODO: parse comma too
|
||||
rcclApiCall call;
|
||||
std::string str(entry);
|
||||
size_t begin = str.find_first_not_of(' ');
|
||||
@@ -588,7 +615,7 @@ void parseJsonEntry(const char* entry, std::vector<rcclApiCall>& calls)
|
||||
case rrCommRegister:
|
||||
{
|
||||
assert(sscanf(str.c_str() + end + 3, (ubr_fmt.substr(5) + ctxt_fmt).c_str(),
|
||||
&call.comm, &call.sendbuff, &call.recvbuff, &call.count) == 4);
|
||||
&call.comm, &call.sendbuff, &call.sendPtrBase, &call.sendPtrExtent, &call.recvbuff, &call.count) == 6);
|
||||
break;
|
||||
}
|
||||
case rrCommDeregister:
|
||||
@@ -658,9 +685,10 @@ void parseJsonEntry(const char* entry, std::vector<rcclApiCall>& calls)
|
||||
}
|
||||
default:
|
||||
assert(sscanf(str.c_str() + end + 3, (coll_fmt.substr(5) + ctxt_fmt).c_str(),
|
||||
&call.opCount, &call.sendbuff, &call.recvbuff, &call.count, &call.datatype, &call.op, &call.root,
|
||||
&call.opCount, &call.sendbuff, &call.sendPtrBase, &call.sendPtrExtent, &call.recvbuff, &call.recvPtrBase, &call.recvPtrExtent,
|
||||
&call.count, &call.datatype, &call.op, &call.root,
|
||||
&call.comm, &call.nRanks, &call.stream, &call.nTasks, &call.globalRank, &call.timestamp, &call.tid,
|
||||
&call.hipDev, &call.graphCaptured, &call.graphID) == 17);
|
||||
&call.hipDev, &call.graphCaptured, &call.graphID) == 21);
|
||||
}
|
||||
calls.push_back(call);
|
||||
}
|
||||
|
||||
@@ -40,16 +40,18 @@ namespace RcclUnitTesting
|
||||
|
||||
std::vector<rccl::rcclApiCall> calls;
|
||||
char entry[4096];
|
||||
gethostname(entry, 256);
|
||||
//parse the outfile
|
||||
std::string filename = "test" + std::to_string(pid) + ".json";
|
||||
std::ifstream fp("/tmp/test" + std::to_string(pid) + ".json");
|
||||
std::string filename = "/tmp/test." + std::to_string(pid) + "." + std::string(entry) + ".json";
|
||||
std::ifstream fp(filename);
|
||||
fp.getline(entry, 4096);
|
||||
fp.getline(entry, 4096);
|
||||
fp.getline(entry, 4096);
|
||||
parseJsonEntry(entry, calls);
|
||||
int result = memcmp((char*)&calls[0]+4, (char*)&call+4, sizeof(rccl::rcclApiCall)-4);
|
||||
// compare only the fields after the pid field
|
||||
int result = memcmp(&(calls[0].pid)+1, &(call.pid)+1, sizeof(rccl::rcclApiCall)-sizeof(call.pid));
|
||||
fp.close(); // care that recorder is not designed to anticipate fp closing before destructor
|
||||
remove(filename.c_str());
|
||||
//remove(filename.c_str()); TODO: after implement fp closing mechanism via envvar, remove file after test case
|
||||
unsetenv("RCCL_REPLAY_FILE");
|
||||
assert(!result);
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ INCLUDES = -I$(MPI_DIR)/include -I$(RCCL_DIR)/include -I$(RCCL_DIR)/hipify/src/i
|
||||
LDFLAGS = -L$(MPI_DIR)/lib -L$(RCCL_DIR) -lmpi -lrccl
|
||||
|
||||
main: rcclReplayer.cpp
|
||||
$(ROCM_DIR)/bin/hipcc rcclReplayer.cpp -O1 -g -o rcclReplayer $(INCLUDES) $(LDFLAGS)
|
||||
$(ROCM_DIR)/bin/hipcc -std=c++20 rcclReplayer.cpp -O0 -g -o rcclReplayer $(INCLUDES) $(LDFLAGS)
|
||||
|
||||
clean:
|
||||
rm -f ./rcclReplayer
|
||||
|
||||
+626
-713
File diff suppressed because it is too large
Load Diff
@@ -2,6 +2,8 @@
|
||||
|
||||
#pragma once
|
||||
#include <map>
|
||||
#include <unordered_map>
|
||||
#include <chrono>
|
||||
#include <cstring>
|
||||
|
||||
#include <rccl/rccl.h>
|
||||
@@ -9,6 +11,9 @@
|
||||
#include "hip/hip_fp16.h"
|
||||
#include "rccl_float8.h"
|
||||
|
||||
#include "info.h"
|
||||
#include "recorder.h"
|
||||
|
||||
// NOTE: Parsing is based on this line logging collective information in enqueue.cc
|
||||
// INFO(NCCL_COLL,"%s: opCount %lx sendbuff %p recvbuff %p count %zi datatype %d op %d \
|
||||
root %d comm %p [nranks=%d] stream %p task %d globalrank %d",
|
||||
@@ -36,396 +41,64 @@
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
struct LineItem
|
||||
struct DeviceMemAllocation
|
||||
{
|
||||
char hostname[MPI_MAX_PROCESSOR_NAME];
|
||||
int pid;
|
||||
int tid;
|
||||
int cudaDev;
|
||||
char opName[32];
|
||||
int opCount;
|
||||
char sendbuff[32];
|
||||
char recvbuff[32];
|
||||
size_t count;
|
||||
int datatype;
|
||||
int op;
|
||||
int root;
|
||||
char comm[32];
|
||||
int nRanks;
|
||||
void* stream;
|
||||
int task;
|
||||
int globalRank;
|
||||
void* base = NULL;
|
||||
size_t size = 0;
|
||||
int lastLineUsed = -1;
|
||||
};
|
||||
|
||||
// Enumeration of all collective functions currently supported
|
||||
typedef enum
|
||||
struct DeviceGraphInfo
|
||||
{
|
||||
ncclCollBroadcast = 0,
|
||||
ncclCollReduce,
|
||||
ncclCollAllGather,
|
||||
ncclCollReduceScatter,
|
||||
ncclCollAllReduce,
|
||||
ncclCollGather,
|
||||
ncclCollScatter,
|
||||
ncclCollAllToAll,
|
||||
ncclCollAllToAllv,
|
||||
ncclCollSend,
|
||||
ncclCollRecv,
|
||||
ncclNumFuncs
|
||||
} ncclFunc_t;
|
||||
int depth = 0;
|
||||
std::unordered_set<int>
|
||||
starts;
|
||||
int end = -1;
|
||||
hipStream_t stream = NULL;
|
||||
hipGraph_t graph = NULL;
|
||||
hipGraphExec_t graphExec = NULL;
|
||||
|
||||
char const ncclFuncNames[ncclNumFuncs][32] =
|
||||
{
|
||||
"Broadcast",
|
||||
"Reduce",
|
||||
"AllGather",
|
||||
"ReduceScatter",
|
||||
"AllReduce",
|
||||
"Gather",
|
||||
"Scatter",
|
||||
"AllToAll",
|
||||
"AllToAllv",
|
||||
"Send",
|
||||
"Recv"
|
||||
std::vector<hipEvent_t>
|
||||
events;
|
||||
int counter = 0;
|
||||
};
|
||||
|
||||
char const mscclFuncNames[ncclNumFuncs][32] =
|
||||
class Replayer
|
||||
{
|
||||
"mscclFuncBroadcast",
|
||||
"mscclFuncReduce",
|
||||
"mscclFuncAllGather",
|
||||
"mscclFuncReduceScatter",
|
||||
"mscclFuncAllReduce",
|
||||
"mscclFuncGather",
|
||||
"mscclFuncScatter",
|
||||
"mscclFuncAllToAll",
|
||||
"mscclFuncAllToAllv",
|
||||
"mscclFuncSend",
|
||||
"mscclFuncRecv"
|
||||
};
|
||||
private:
|
||||
// rank specific info
|
||||
int myRank;
|
||||
int numGlobalRanks;
|
||||
/// int numGpusPerMpiRank;
|
||||
/// int localGpuOffset; // First local GPU device idx for this MPI process
|
||||
/// int firstGlobalRank; // First global rank for this MPI process
|
||||
std::ifstream log;
|
||||
|
||||
union PtrUnion
|
||||
{
|
||||
void* ptr;
|
||||
int8_t* I1; // ncclInt8
|
||||
uint8_t* U1; // ncclUint8
|
||||
int32_t* I4; // ncclInt32
|
||||
uint32_t* U4; // ncclUint32
|
||||
int64_t* I8; // ncclInt64
|
||||
uint64_t* U8; // ncclUint64
|
||||
__half* F2; // ncclFloat16
|
||||
rccl_float8* F1; // ncclFloat8e4m3
|
||||
float* F4; // ncclFloat32
|
||||
double* F8; // ncclFloat64
|
||||
rccl_bfloat8* B1; // ncclFloat8e5m2
|
||||
hip_bfloat16* B2; // ncclBfloat16
|
||||
// Contextual info parsed from first pass, to assist replay later
|
||||
// Communicator
|
||||
std::vector<uint64_t> Ids; // all communicators (uniqueIDs) created from commInit, assuming called once only
|
||||
std::unordered_map<uint64_t, std::vector<int>> idRankMap; // all ranks in the communicator created by an ID on this rank
|
||||
|
||||
constexpr PtrUnion() : ptr(nullptr) {}
|
||||
};
|
||||
|
||||
struct TaskInfo
|
||||
{
|
||||
ncclFunc_t funcType;
|
||||
bool inPlace;
|
||||
size_t count;
|
||||
ncclDataType_t datatype;
|
||||
ncclRedOp_t op;
|
||||
int root;
|
||||
PtrUnion inputGpu;
|
||||
PtrUnion outputCpu;
|
||||
PtrUnion outputGpu;
|
||||
PtrUnion expected;
|
||||
};
|
||||
// Memory allocation and lifespan
|
||||
std::unordered_map<void*, DeviceMemAllocation> dMemMap;
|
||||
|
||||
struct RankData
|
||||
{
|
||||
int lineNum;
|
||||
int commIdx;
|
||||
std::vector<TaskInfo> tasks;
|
||||
};
|
||||
// Resources for replayer, mostly maps from pointer in log to resources in replay time
|
||||
std::unordered_map<uint64_t, ncclUniqueId> idMap; // replayer uniqueID mapped to logged ones, for ID creator rank only
|
||||
std::unordered_map<ncclComm_t, ncclComm_t> commMap; // replayer communicator mapped to the logged ones
|
||||
|
||||
struct GroupCall
|
||||
{
|
||||
bool isValid;
|
||||
int opCount;
|
||||
std::map<int, RankData> rankData;
|
||||
};
|
||||
std::unordered_map<hipStream_t, std::pair<hipStream_t,int>>
|
||||
streams; // replayer streams mapped to the logged ones // use using later?
|
||||
std::unordered_map<void*, void*> handleMap; // UBR handle
|
||||
std::unordered_map<unsigned long long, DeviceGraphInfo>
|
||||
graphLife; // when does a graph (graphID) end and how many node it contains
|
||||
|
||||
struct CollectiveCalls
|
||||
{
|
||||
int numGlobalRanks;
|
||||
int numGpusPerMpiRank;
|
||||
std::vector<std::vector<std::string>> globalRankComms; // Set of comms used by each global rank
|
||||
std::vector<GroupCall> groupCalls; // List of group calls for each global rank
|
||||
// auxiliary variables for replayer
|
||||
ncclUniqueId uniqueID;
|
||||
rccl::rcclCall_t lastCall;
|
||||
|
||||
int localGpuOffset; // First local GPU device idx for this MPI process
|
||||
int firstGlobalRank; // First global rank for this MPI process
|
||||
int numCommsPerRank; // Number of communicators per rank
|
||||
std::vector<std::vector<ncclComm_t>> localRankComms; // comms per local rank
|
||||
std::vector<std::vector<hipStream_t>> localRankStreams; // streams per local rank
|
||||
};
|
||||
|
||||
std::string DataTypeToName(ncclDataType_t const dataType)
|
||||
{
|
||||
switch (dataType) {
|
||||
case ncclInt8: return "Int8";
|
||||
case ncclUint8: return "Uint8";
|
||||
case ncclInt32: return "Int32";
|
||||
case ncclUint32: return "Uint32";
|
||||
case ncclInt64: return "Int64";
|
||||
case ncclUint64: return "Uint64";
|
||||
case ncclFloat16: return "Float16";
|
||||
case ncclFloat32: return "Float32";
|
||||
case ncclFloat64: return "Float64";
|
||||
case ncclBfloat16: return "Bfloat16";
|
||||
case ncclFloat8e4m3: return "Fp8E4M3";
|
||||
case ncclFloat8e5m2: return "Fp8E5M2";
|
||||
default:
|
||||
printf("Unsupported datatype (%d)\n", dataType);
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
size_t DataTypeToBytes(ncclDataType_t const dataType)
|
||||
{
|
||||
switch (dataType) {
|
||||
case ncclInt8: return 1;
|
||||
case ncclUint8: return 1;
|
||||
case ncclInt32: return 4;
|
||||
case ncclUint32: return 4;
|
||||
case ncclInt64: return 8;
|
||||
case ncclUint64: return 8;
|
||||
case ncclFloat16: return 2;
|
||||
case ncclFloat32: return 4;
|
||||
case ncclFloat64: return 8;
|
||||
case ncclBfloat16: return 2;
|
||||
case ncclFloat8e4m3: return 1;
|
||||
case ncclFloat8e5m2: return 1;
|
||||
default:
|
||||
printf("Unsupported datatype (%s)\n", DataTypeToName(dataType).c_str());
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
std::string RedOpToName(ncclRedOp_t const op)
|
||||
{
|
||||
switch (op) {
|
||||
case ncclSum: return "Sum";
|
||||
case ncclProd: return "Product";
|
||||
case ncclMax: return "Max";
|
||||
case ncclMin: return "Min";
|
||||
case ncclAvg: return "Average";
|
||||
case ncclNumOps: return "Number of built-in reduction ops";
|
||||
case ncclMaxRedOp: return "Largest value for ncclRedOp_t";
|
||||
default:
|
||||
printf("Unsupported redOp (%d)\n", op);
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
ncclFunc_t GetFuncType(char* func)
|
||||
{
|
||||
for (int i = 0; i < ncclNumFuncs; i++)
|
||||
if (!strcmp(func, ncclFuncNames[i]) || !strcmp(func, mscclFuncNames[i])) return (ncclFunc_t)i;
|
||||
printf("[ERROR] Unrecognized func %s\n", func);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
// Set data for ptrUnion (Used during fillPattern)
|
||||
void SetPtr(PtrUnion& ptrUnion, ncclDataType_t const dataType, int const idx, int valueI, double valueF) {
|
||||
switch (dataType)
|
||||
{
|
||||
case ncclInt8: ptrUnion.I1[idx] = valueI; break;
|
||||
case ncclUint8: ptrUnion.U1[idx] = valueI; break;
|
||||
case ncclInt32: ptrUnion.I4[idx] = valueI; break;
|
||||
case ncclUint32: ptrUnion.U4[idx] = valueI; break;
|
||||
case ncclInt64: ptrUnion.I8[idx] = valueI; break;
|
||||
case ncclUint64: ptrUnion.U8[idx] = valueI; break;
|
||||
case ncclFloat8e4m3: ptrUnion.F1[idx] = rccl_float8(valueF); break;
|
||||
case ncclFloat16: ptrUnion.F2[idx] = __float2half(static_cast<float>(valueF)); break;
|
||||
case ncclFloat32: ptrUnion.F4[idx] = valueF; break;
|
||||
case ncclFloat64: ptrUnion.F8[idx] = valueF; break;
|
||||
case ncclFloat8e5m2: ptrUnion.B1[idx] = rccl_bfloat8(valueF); break;
|
||||
case ncclBfloat16: ptrUnion.B2[idx] = hip_bfloat16(static_cast<float>(valueF)); break;
|
||||
default:
|
||||
printf("Unsupported datatype (%s)\n", DataTypeToName(dataType).c_str());
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Check if each element in actual equals to expected
|
||||
bool IsEqual(PtrUnion const& actual, PtrUnion const& expected, ncclDataType_t const dataType, size_t const numElements, int const globalRank) {
|
||||
bool isMatch = true;
|
||||
size_t idx = 0;
|
||||
for (idx = 0; idx < numElements; ++idx)
|
||||
{
|
||||
switch (dataType)
|
||||
{
|
||||
case ncclInt8: isMatch = (actual.I1[idx] == expected.I1[idx]); break;
|
||||
case ncclUint8: isMatch = (actual.U1[idx] == expected.U1[idx]); break;
|
||||
case ncclInt32: isMatch = (actual.I4[idx] == expected.I4[idx]); break;
|
||||
case ncclUint32: isMatch = (actual.U4[idx] == expected.U4[idx]); break;
|
||||
case ncclInt64: isMatch = (actual.I8[idx] == expected.I8[idx]); break;
|
||||
case ncclUint64: isMatch = (actual.U8[idx] == expected.U8[idx]); break;
|
||||
case ncclFloat8e4m3: isMatch = (fabs(float(actual.F1[idx]) - float(expected.F1[idx])) < 9e-2); break;
|
||||
case ncclFloat16: isMatch = (fabs(__half2float(actual.F2[idx]) - __half2float(expected.F2[idx])) < 9e-2); break;
|
||||
case ncclFloat32: isMatch = (fabs(actual.F4[idx] - expected.F4[idx]) < 1e-5); break;
|
||||
case ncclFloat64: isMatch = (fabs(actual.F8[idx] - expected.F8[idx]) < 1e-12); break;
|
||||
case ncclFloat8e5m2: isMatch = (fabs(float(actual.B1[idx]) - float(expected.B1[idx])) < 9e-2); break;
|
||||
case ncclBfloat16: isMatch = (fabs((float)actual.B2[idx] - (float)expected.B2[idx]) < 9e-2); break;
|
||||
default:
|
||||
printf("Unsupported datatype (%s)\n", DataTypeToName(dataType).c_str());
|
||||
isMatch = false;
|
||||
}
|
||||
if (!isMatch) {
|
||||
switch (dataType)
|
||||
{
|
||||
case ncclInt8:
|
||||
printf("[Error Rank = %d] Expected output: %d. Actual output: %d at index %lu\n", globalRank, expected.I1[idx], actual.I1[idx], idx); break;
|
||||
case ncclUint8:
|
||||
printf("[Error Rank = %d] Expected output: %u. Actual output: %u at index %lu\n", globalRank, expected.U1[idx], actual.U1[idx], idx); break;
|
||||
case ncclInt32:
|
||||
printf("[Error Rank = %d] Expected output: %d. Actual output: %d at index %lu\n", globalRank, expected.I4[idx], actual.I4[idx], idx); break;
|
||||
case ncclUint32:
|
||||
printf("[Error Rank = %d] Expected output: %u. Actual output: %u at index %lu\n", globalRank, expected.U4[idx], actual.U4[idx], idx); break;
|
||||
case ncclInt64:
|
||||
printf("[Error Rank = %d] Expected output: %ld. Actual output: %ld at index %lu\n", globalRank, expected.I8[idx], actual.I8[idx], idx); break;
|
||||
case ncclUint64:
|
||||
printf("[Error Rank = %d] Expected output: %lu. Actual output: %lu at index %lu\n", globalRank, expected.U8[idx], actual.U8[idx], idx); break;
|
||||
case ncclFloat8e4m3:
|
||||
printf("[Error Rank = %d] Expected output: %f. Actual output: %f at index %lu\n", globalRank, (float)expected.F1[idx], (float)actual.F1[idx], idx); break;
|
||||
case ncclFloat16:
|
||||
printf("[Error Rank = %d] Expected output: %f. Actual output: %f at index %lu\n", globalRank, __half2float(expected.F2[idx]), __half2float(actual.F2[idx]), idx); break;
|
||||
case ncclFloat32:
|
||||
printf("[Error Rank = %d] Expected output: %f. Actual output: %f at index %lu\n", globalRank, expected.F4[idx], actual.F4[idx], idx); break;
|
||||
case ncclFloat64:
|
||||
printf("[Error Rank = %d] Expected output: %lf. Actual output: %lf at index %lu\n", globalRank, expected.F8[idx], actual.F8[idx], idx); break;
|
||||
case ncclFloat8e5m2:
|
||||
printf("[Error Rank = %d] Expected output: %f. Actual output: %f at index %lu\n", globalRank, (float)expected.B1[idx], (float)actual.B1[idx], idx); break;
|
||||
case ncclBfloat16:
|
||||
printf("[Error Rank = %d] Expected output: %f. Actual output: %f at index %lu\n", globalRank, (float)expected.B2[idx], (float)actual.B2[idx], idx); break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return isMatch;
|
||||
}
|
||||
}
|
||||
|
||||
return isMatch;
|
||||
}
|
||||
|
||||
// Performs the various basic reduction operations
|
||||
template <typename T>
|
||||
T ReduceOp(ncclRedOp_t const op, T const A, T const B)
|
||||
{
|
||||
switch (op)
|
||||
{
|
||||
case ncclSum: return A + B;
|
||||
case ncclProd: return A * B;
|
||||
case ncclMax: return std::max(A, B);
|
||||
case ncclMin: return std::min(A, B);
|
||||
default:
|
||||
printf("Unsupported reduction operator (%s)\n", RedOpToName(op).c_str());
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
// Perform various reduction ops to ptrUnion
|
||||
void Reduce(PtrUnion& ptrUnion, PtrUnion const& otherPtrUnion, size_t const numElements, ncclDataType_t const dataType, ncclRedOp_t const op) {
|
||||
for (size_t idx = 0; idx < numElements; ++idx)
|
||||
{
|
||||
switch (dataType)
|
||||
{
|
||||
case ncclInt8: ptrUnion.I1[idx] = ReduceOp(op, ptrUnion.I1[idx], otherPtrUnion.I1[idx]); break;
|
||||
case ncclUint8: ptrUnion.U1[idx] = ReduceOp(op, ptrUnion.U1[idx], otherPtrUnion.U1[idx]); break;
|
||||
case ncclInt32: ptrUnion.I4[idx] = ReduceOp(op, ptrUnion.I4[idx], otherPtrUnion.I4[idx]); break;
|
||||
case ncclUint32: ptrUnion.U4[idx] = ReduceOp(op, ptrUnion.U4[idx], otherPtrUnion.U4[idx]); break;
|
||||
case ncclInt64: ptrUnion.I8[idx] = ReduceOp(op, ptrUnion.I8[idx], otherPtrUnion.I8[idx]); break;
|
||||
case ncclUint64: ptrUnion.U8[idx] = ReduceOp(op, ptrUnion.U8[idx], otherPtrUnion.U8[idx]); break;
|
||||
case ncclFloat8e4m3: ptrUnion.F1[idx] = rccl_float8(ReduceOp(op, float(ptrUnion.F1[idx]), float(otherPtrUnion.F1[idx]))); break;
|
||||
case ncclFloat16: ptrUnion.F2[idx] = __float2half(ReduceOp(op, __half2float(ptrUnion.F2[idx]), __half2float(otherPtrUnion.F2[idx]))); break;
|
||||
case ncclFloat32: ptrUnion.F4[idx] = ReduceOp(op, ptrUnion.F4[idx], otherPtrUnion.F4[idx]); break;
|
||||
case ncclFloat64: ptrUnion.F8[idx] = ReduceOp(op, ptrUnion.F8[idx], otherPtrUnion.F8[idx]); break;
|
||||
case ncclFloat8e5m2: ptrUnion.B1[idx] = rccl_bfloat8(ReduceOp(op, float(ptrUnion.B1[idx]), float(otherPtrUnion.B1[idx]))); break;
|
||||
case ncclBfloat16: ptrUnion.B2[idx] = ReduceOp(op, ptrUnion.B2[idx], otherPtrUnion.B2[idx]); break;
|
||||
default:
|
||||
printf("Unsupported datatype (%s)\n", DataTypeToName(dataType).c_str());
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Divide each element in ptrUnion by divisor
|
||||
void DivideByInt(PtrUnion& ptrUnion, ncclDataType_t const dataType, size_t const numElements, int const divisor) {
|
||||
for (size_t idx = 0; idx < numElements; ++idx)
|
||||
{
|
||||
switch (dataType)
|
||||
{
|
||||
case ncclInt8: ptrUnion.I1[idx] /= divisor; break;
|
||||
case ncclUint8: ptrUnion.U1[idx] /= divisor; break;
|
||||
case ncclInt32: ptrUnion.I4[idx] /= divisor; break;
|
||||
case ncclUint32: ptrUnion.U4[idx] /= divisor; break;
|
||||
case ncclInt64: ptrUnion.I8[idx] /= divisor; break;
|
||||
case ncclUint64: ptrUnion.U8[idx] /= divisor; break;
|
||||
case ncclFloat8e4m3: ptrUnion.F1[idx] = (rccl_float8((float)(ptrUnion.F1[idx]) / divisor)); break;
|
||||
case ncclFloat16: ptrUnion.F2[idx] = __float2half(__half2float(ptrUnion.F2[idx])/divisor); break;
|
||||
case ncclFloat32: ptrUnion.F4[idx] /= divisor; break;
|
||||
case ncclFloat64: ptrUnion.F8[idx] /= divisor; break;
|
||||
case ncclFloat8e5m2: ptrUnion.B1[idx] = (rccl_bfloat8((float)(ptrUnion.B1[idx]) / divisor)); break;
|
||||
case ncclBfloat16: ptrUnion.B2[idx] = (hip_bfloat16((float)(ptrUnion.B2[idx]) / divisor)); break;
|
||||
default:
|
||||
printf("Unsupported datatype (%s)\n", DataTypeToName(dataType).c_str());
|
||||
exit(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check if a collective uses a root
|
||||
bool IsRootUsed(ncclFunc_t funcType) {
|
||||
return (funcType == ncclCollBroadcast || funcType == ncclCollReduce ||
|
||||
funcType == ncclCollGather || funcType == ncclCollScatter);
|
||||
}
|
||||
|
||||
// parse the logs and assign them into lineItem
|
||||
bool ParseLineItem(char const* line, LineItem& li);
|
||||
|
||||
// this covers grouping the logs based on opCount and task number,
|
||||
// validatation of the groupCalls for both non-send/recv collectives and send/recv
|
||||
void ParseCollectives(char const* logFilename, bool isFirstRank, CollectiveCalls& collectiveCalls);
|
||||
|
||||
// allocates send/recv buff, sets the device based on which rank the task belongs to,
|
||||
// syncronize devices after executing all the tasks and free device memory.
|
||||
double ReplayRccl(CollectiveCalls& collCall, int groupIdx, int& numInvalid);
|
||||
|
||||
// Print information about a group call
|
||||
void PrintGroupCall(GroupCall const& gc);
|
||||
|
||||
// Records performance data of each group call in a csv file named replayer_data.csv
|
||||
void dataToCsv(GroupCall const& gc, std::ofstream &datafile, double runTime);
|
||||
|
||||
// size differ for each collective call and getSize gives a specific size in bytes depending on type of task,
|
||||
// global rank, element count and data type
|
||||
std::pair<size_t, size_t> GetSize(TaskInfo taskInfo, int numGlobalRanks);
|
||||
|
||||
// executes the collective call (task)
|
||||
void ExecuteCollective(TaskInfo& task, ncclComm_t const& comm, hipStream_t stream);
|
||||
|
||||
// Allocate CPU/GPU memory for ptrUnion
|
||||
void AllocateMem(PtrUnion& ptrUnion, size_t const numBytes, bool isGpu = false);
|
||||
|
||||
// Free CPU/GPU memory for ptrUnion
|
||||
void FreeMem(PtrUnion& ptrUnion, bool isGpu = false);
|
||||
|
||||
// Fill buffers based on pattern using globalRank
|
||||
void FillPattern(PtrUnion& ptrUnion, ncclDataType_t const dataType, size_t const numElements, int globalRank, bool isGpu = false);
|
||||
|
||||
// PrepareData functions are responsible for setting up input / expected for the given taskInfo
|
||||
void PrepareDataFunc(TaskInfo& taskInfo, int globalRank, int totalRanks);
|
||||
void PrepData_Broadcast(TaskInfo& taskInfo, int globalRank);
|
||||
void PrepData_Reduce(TaskInfo& taskInfo, int globalRank, int totalRanks, bool isAllReduce);
|
||||
void PrepData_ReduceScatter(TaskInfo& taskInfo, int globalRank, int totalRanks);
|
||||
void PrepData_Gather(TaskInfo& taskInfo, int globalRank, int totalRanks, bool isAllGather);
|
||||
void PrepData_Scatter(TaskInfo& taskInfo, int globalRank, int totalRanks);
|
||||
void PrepData_AlltoAll(TaskInfo& taskInfo, int globalRank, int totalRanks);
|
||||
void PrepData_Send(TaskInfo& taskInfo, int globalRank);
|
||||
void PrepData_Recv(TaskInfo& taskInfo, int globalRank);
|
||||
public:
|
||||
Replayer(const std::string& logname, int json_format, int rank, int size);
|
||||
void parse();
|
||||
void replay();
|
||||
};
|
||||
Reference in New Issue
Block a user