AI/rocm-systems
2
0
派生 0
代码 工单 合并请求 工作流 软件包 项目 发布 百科 活动

RCCL Replayer update (#1603)

浏览代码 
RCCL recorder w/ suggested change and UT
这个提交包含在：
Tim
2025-04-19 00:21:27 -04:00
提交者 GitHub
父节点 52bfdf05dc
当前提交 9a55ff60a9
共修改 18 个文件，包含 1123 行新增和 93 行删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件
tools/RcclReplayer/Makefile
+12
查看文件
@@ -0,0 +1,12 @@
ROCM_DIR ?= /opt/rocm
RCCL_DIR ?= ../../build/release
MPI_DIR ?= /opt/ompi
INCLUDES = -I$(MPI_DIR)/include -I$(RCCL_DIR)/include -I$(RCCL_DIR)/hipify/src/include
LDFLAGS = -L$(MPI_DIR)/lib -L$(RCCL_DIR) -lmpi -lrccl
main: rcclReplayer.cpp
$(ROCM_DIR)/bin/hipcc rcclReplayer.cpp -O1 -g -o rcclReplayer $(INCLUDES) $(LDFLAGS)
clean:
rm -f ./rcclReplayer
tools/rccl_replayer/README.md → tools/RcclReplayer/README.md
查看文件
tools/rccl_replayer/rcclReplayer.cpp → tools/RcclReplayer/rcclReplayer.cpp
+27 -28
查看文件
@@ -180,31 +180,36 @@ void dataToCsv(GroupCall const& gc, std::ofstream &datafile, double runTime)
void ParseCollectives(char const* logFilename, bool isFirstRank, CollectiveCalls& cc)
{
bool verbose = isFirstRank && (getenv("VERBOSE") != NULL);
cc.globalRankComms.clear();
cc.globalRankComms.resize(cc.numGlobalRanks);
cc.groupCalls.clear();
FILE* fp = fopen(logFilename, "r");
std::ifstream fp(logFilename, std::ios::binary);
if (!fp) {
printf("[ERROR] Unable to open file %s\n", logFilename);
exit(-1);
}
char line[2048];
LineItem li;
constexpr size_t size = sizeof(LineItem) + 1;
char line[size]; //size of collectivecall struct
int lineNum = 0;
while (fgets(line, 2048, fp)) {
++lineNum;
while (fp.getline(line, size))
{
LineItem li = *((LineItem*)line);
//Ignore invalid lines and collectives
if (!ParseLineItem(line, li) || li.nRanks != cc.numGlobalRanks) continue;
if (li.coll == 10 || li.coll == 11)
{
continue;
}
// if need hostname will parse together with num of line in output
// Ignore invalid lines and collectives
if (li.nRanks != cc.numGlobalRanks) continue;
// Figure out commIdx for this globalrank
int commIdx = -1;
for (auto i = 0; i < cc.globalRankComms[li.globalRank].size(); i++) {
if (!strcmp(cc.globalRankComms[li.globalRank][i].c_str(), li.comm)) {
if (cc.globalRankComms[li.globalRank][i] != li.comm) {
commIdx = i;
break;
}
@@ -215,8 +220,8 @@ void ParseCollectives(char const* logFilename, bool isFirstRank, CollectiveCalls
}
TaskInfo taskInfo;
taskInfo.funcType = GetFuncType(li.opName);
taskInfo.inPlace = !strcmp(li.sendbuff, li.recvbuff);
taskInfo.funcType = (ncclFunc_t) li.coll;
taskInfo.inPlace = li.sendbuff == li.recvbuff;
taskInfo.count = li.count;
taskInfo.datatype = (ncclDataType_t) li.datatype;
taskInfo.op = (ncclRedOp_t) li.op;
@@ -229,7 +234,7 @@ void ParseCollectives(char const* logFilename, bool isFirstRank, CollectiveCalls
if (gc.opCount != li.opCount) continue;
if (gc.rankData.count(li.globalRank)) {
RankData& rd = gc.rankData[li.globalRank];
if (rd.commIdx != commIdx || rd.tasks.size() != li.task)
if (rd.commIdx != commIdx || rd.tasks.size() != li.nTasks)
continue;
rd.tasks.push_back(taskInfo);
@@ -237,7 +242,7 @@ void ParseCollectives(char const* logFilename, bool isFirstRank, CollectiveCalls
break;
}
// Rank has no tasks - make sure this is task 0
else if (li.task == 0) {
else if (li.nTasks == 0) {
gc.rankData[li.globalRank].lineNum = lineNum;
gc.rankData[li.globalRank].commIdx = commIdx;
gc.rankData[li.globalRank].tasks.push_back(taskInfo);
@@ -248,7 +253,7 @@ void ParseCollectives(char const* logFilename, bool isFirstRank, CollectiveCalls
// If no collectives were found, create new one
if (!found) {
if (li.task != 0) {
if (li.nTasks != 0) {
if (isFirstRank) printf("[WARN] Was unable to find corresponding collective for line %d\n", lineNum);
}
@@ -260,7 +265,7 @@ void ParseCollectives(char const* logFilename, bool isFirstRank, CollectiveCalls
cc.groupCalls.push_back(gc);
}
}
fclose(fp);
fp.close();
// Validate group calls
// - For non Send/Recv, check that all ranks participate with same parameters count
@@ -375,18 +380,6 @@ void ParseCollectives(char const* logFilename, bool isFirstRank, CollectiveCalls
}
}
bool ParseLineItem(char const* line, LineItem& li)
{
return sscanf(line,
"%[^:]:%d:%d [%d] NCCL INFO %[^:]: opCount %x sendbuff %s "
"recvbuff %s count %lu datatype %d op %d root %d comm %s "
"[nranks=%d] stream %p task %d globalrank %d",
li.hostname, &li.pid, &li.tid, &li.cudaDev, li.opName,
&li.opCount, li.sendbuff, li.recvbuff,
&li.count, &li.datatype, &li.op, &li.root, li.comm,
&li.nRanks, &li.stream, &li.task, &li.globalRank) == 17;
}
double ReplayRccl(CollectiveCalls& cc, int groupIdx, int& numInvalid)
{
int numLocalRanks = cc.localRankComms.size();
@@ -566,6 +559,12 @@ void ExecuteCollective(TaskInfo& task, ncclComm_t const& comm, hipStream_t strea
case ncclCollRecv:
NCCL_CALL(ncclRecv(task.outputGpu.ptr, task.count, task.datatype, task.root, comm, stream));
break;
case ncclStartGroup:
NCCL_CALL(ncclGroupStart());
break;
case ncclEndGroup:
NCCL_CALL(ncclGroupEnd());
break;
default:
printf("Error: unsupported collective\n");
exit(1);
@@ -755,4 +754,4 @@ void PrepData_Send(TaskInfo& taskInfo, int globalRank) {
void PrepData_Recv(TaskInfo& taskInfo, int globalRank) {
FillPattern(taskInfo.expected, taskInfo.datatype, taskInfo.count, globalRank);
}
}
tools/rccl_replayer/rcclReplayer.hpp → tools/RcclReplayer/rcclReplayer.hpp
+27 -25
查看文件
@@ -1,5 +1,6 @@
#pragma once
#include <map>
#include <chrono>
#include <cstring>
#include <rccl/rccl.h>
@@ -36,23 +37,25 @@
struct LineItem
{
char hostname[MPI_MAX_PROCESSOR_NAME];
int pid;
int tid;
int cudaDev;
char opName[32];
int opCount;
char sendbuff[32];
char recvbuff[32];
size_t count;
int datatype;
int op;
int root;
char comm[32];
int nRanks;
void* stream;
int task;
int globalRank;
int pid;
int tid;
int cudaDev;
//int graph;
int groupDepth;
int coll;
uint64_t opCount;
void* sendbuff;
void* recvbuff;
size_t count;
int datatype;
int op;
int root;
void* comm;
int nRanks;
void* stream;
int nTasks;
int globalRank;
};
// Enumeration of all collective functions currently supported
@@ -69,7 +72,9 @@ typedef enum
ncclCollAllToAllv,
ncclCollSend,
ncclCollRecv,
ncclNumFuncs
ncclNumFuncs,
ncclStartGroup = 10;
ncclEndGroup = 11;
} ncclFunc_t;
char const ncclFuncNames[ncclNumFuncs][32] =
@@ -144,8 +149,8 @@ struct RankData
struct GroupCall
{
bool isValid;
int opCount;
bool isValid;
uint64_t opCount;
std::map<int, RankData> rankData;
};
@@ -153,7 +158,7 @@ struct CollectiveCalls
{
int numGlobalRanks;
int numGpusPerMpiRank;
std::vector<std::vector<std::string>> globalRankComms; // Set of comms used by each global rank
std::vector<std::vector<void*>> globalRankComms; // Set of comms used by each global rank
std::vector<GroupCall> groupCalls; // List of group calls for each global rank
int localGpuOffset; // First local GPU device idx for this MPI process
@@ -384,9 +389,6 @@ bool IsRootUsed(ncclFunc_t funcType) {
funcType == ncclCollGather || funcType == ncclCollScatter);
}
// parse the logs and assign them into lineItem
bool ParseLineItem(char const* line, LineItem& li);
// this covers grouping the logs based on opCount and task number,
// validatation of the groupCalls for both non-send/recv collectives and send/recv
void ParseCollectives(char const* logFilename, bool isFirstRank, CollectiveCalls& collectiveCalls);
@@ -426,4 +428,4 @@ void PrepData_Gather(TaskInfo& taskInfo, int globalRank, int totalRanks, bool is
void PrepData_Scatter(TaskInfo& taskInfo, int globalRank, int totalRanks);
void PrepData_AlltoAll(TaskInfo& taskInfo, int globalRank, int totalRanks);
void PrepData_Send(TaskInfo& taskInfo, int globalRank);
void PrepData_Recv(TaskInfo& taskInfo, int globalRank);
void PrepData_Recv(TaskInfo& taskInfo, int globalRank);
tools/rccl_replayer/Makefile
-14
查看文件
@@ -1,14 +0,0 @@
ROCM_DIR ?= /opt/rocm
RCCL_DIR ?= ../../build/release
MPI_DIR ?= /opt/ompi
MPI_INC_DIR ?= /usr/include/x86_64-linux-gnu/mpi
MPI_LIB_DIR ?= /usr/lib/x86_64-linux-gnu
INCLUDES = -I$(MPI_INC_DIR) -I$(MPI_DIR)/include -I$(RCCL_DIR)/include -I$(RCCL_DIR)/hipify/src/include
LDFLAGS = -L$(MPI_LIB_DIR) -L$(MPI_DIR)/lib -L$(RCCL_DIR) -lmpi -lrccl
main: rcclReplayer.cpp
$(ROCM_DIR)/bin/hipcc rcclReplayer.cpp -O1 -g -o rcclReplayer $(INCLUDES) $(LDFLAGS)
clean:
rm -f ./rcclReplayer