AI/rocm-systems
2
0
Форкнуть 0
Код Задачи Запросы на слияние Действия Пакеты Проекты Релизы Вики Активность

Adjustment for UT Sendrecv (#1400)

Просмотр исходного кода 
Enabled UT sendrecv to same rank and refactor UBR call
Этот коммит содержится в:
Tim
2024-10-30 15:13:53 -04:00
коммит произвёл GitHub
родитель 0fb3b5eba9
Коммит fd9924cfe7
2 изменённых файлов: 83 добавлений и 56 удалений
Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы
test/SendRecvTests.cpp
+77 -50
Просмотреть файл
@@ -16,7 +16,6 @@ namespace RcclUnitTesting
std::vector<int> const numElements = {1048576, 53327, 1024, 0};
bool const inPlace = false;
bool const useManagedMem = false;
int const groupCallId = 0;
OptionalColArgs options;
bool isCorrect = true;
@@ -28,7 +27,10 @@ namespace RcclUnitTesting
int ranksPerGpu = rpg == 0 ? 1 : testBed.ev.maxRanksPerGpu;
int totalRanks = numGpus * ranksPerGpu;
int const numProcesses = isMultiProcess ? numGpus : 1;
testBed.InitComms(TestBed::GetDeviceIdsList(numProcesses, numGpus, ranksPerGpu), 1);
testBed.InitComms(TestBed::GetDeviceIdsList(numProcesses, numGpus, ranksPerGpu),
{1,2}, //two group, second group sendrecv to self, has 2 coll
testBed.GetNumStreamsPerGroup(1,2),
2);
for (int dataIdx = 0; dataIdx < dataTypes.size() && isCorrect; ++dataIdx)
for (int numIdx = 0; numIdx < numElements.size() && isCorrect; ++numIdx)
@@ -37,6 +39,8 @@ namespace RcclUnitTesting
for (int recvRank = 0; recvRank < totalRanks; ++recvRank)
{
options.root = recvRank;
int groupCallId = sendRank == recvRank; //self sendrecv group has two coll
int recvId = sendRank == recvRank; //where recv will be second coll
testBed.SetCollectiveArgs(ncclCollSend,
dataTypes[dataIdx],
numElements[numIdx],
@@ -47,36 +51,46 @@ namespace RcclUnitTesting
sendRank);
if (recvRank == 0)
{
testBed.AllocateMem(inPlace, useManagedMem, groupCallId, 0, sendRank);
testBed.PrepareData(groupCallId, 0, sendRank);
}
if (recvRank != sendRank)
{
if (testBed.ev.showNames) // Show test names
INFO("%s Datatype: %s SendReceive test Rank %d -> Rank %d for %d Elements\n",
isMultiProcess ? "MP" : "SP",
ncclDataTypeNames[dataTypes[dataIdx]],
sendRank,
recvRank,
numElements[numIdx]);
options.root = sendRank;
testBed.SetCollectiveArgs(ncclCollRecv,
//set up the collArg slot to make sure AllocateMem is called once and correctly
testBed.SetCollectiveArgs(ncclCollSend,
dataTypes[dataIdx],
numElements[numIdx],
numElements[numIdx],
options,
0,
groupCallId,
recvRank);
testBed.AllocateMem(inPlace, useManagedMem, groupCallId, 0, recvRank);
testBed.PrepareData(groupCallId, 0, recvRank);
testBed.ExecuteCollectives({sendRank, recvRank});
testBed.ValidateResults(isCorrect, groupCallId, 0, recvRank);
testBed.DeallocateMem(groupCallId, 0, recvRank);
!groupCallId,
sendRank);
testBed.AllocateMem(inPlace, useManagedMem, 0, 0, sendRank);
testBed.PrepareData(0, 0, sendRank);
testBed.AllocateMem(inPlace, useManagedMem, 1, 0, sendRank);
testBed.PrepareData(1, 0, sendRank);
}
if (testBed.ev.showNames) // Show test names
INFO("%s Datatype: %s SendReceive test Rank %d -> Rank %d for %d Elements\n",
isMultiProcess ? "MP" : "SP",
ncclDataTypeNames[dataTypes[dataIdx]],
sendRank,
recvRank,
numElements[numIdx]);
options.root = sendRank;
testBed.SetCollectiveArgs(ncclCollRecv,
dataTypes[dataIdx],
numElements[numIdx],
numElements[numIdx],
options,
recvId,
groupCallId,
recvRank);
testBed.AllocateMem(inPlace, useManagedMem, groupCallId, recvId, recvRank);
testBed.PrepareData(groupCallId, recvId, recvRank);
testBed.ExecuteCollectives({sendRank, recvRank}, groupCallId);
testBed.ValidateResults(isCorrect, groupCallId, recvId, recvRank);
testBed.DeallocateMem(groupCallId, recvId, recvRank);
}
testBed.DeallocateMem(groupCallId, 0, sendRank);
testBed.DeallocateMem(0, 0, sendRank);
testBed.DeallocateMem(1, 0, sendRank);
}
testBed.DestroyComms();
}
@@ -94,7 +108,6 @@ namespace RcclUnitTesting
bool const inPlace = false;
bool const useManagedMem = false;
bool const userRegistered = true;
int const groupCallId = 0;
OptionalColArgs options;
bool isCorrect = true;
@@ -106,7 +119,10 @@ namespace RcclUnitTesting
int ranksPerGpu = rpg == 0 ? 1 : testBed.ev.maxRanksPerGpu;
int totalRanks = numGpus * ranksPerGpu;
int const numProcesses = isMultiProcess ? numGpus : 1;
testBed.InitComms(TestBed::GetDeviceIdsList(numProcesses, numGpus, ranksPerGpu), 1);
testBed.InitComms(TestBed::GetDeviceIdsList(numProcesses, numGpus, ranksPerGpu),
{1,2}, //two group, second group sendrecv to self, has 2 coll
testBed.GetNumStreamsPerGroup(1,2),
2);
for (int dataIdx = 0; dataIdx < dataTypes.size() && isCorrect; ++dataIdx)
for (int numIdx = 0; numIdx < numElements.size() && isCorrect; ++numIdx)
@@ -115,6 +131,8 @@ namespace RcclUnitTesting
for (int recvRank = 0; recvRank < totalRanks; ++recvRank)
{
options.root = recvRank;
int groupCallId = sendRank == recvRank;
int recvId = sendRank == recvRank;
testBed.SetCollectiveArgs(ncclCollSend,
dataTypes[dataIdx],
numElements[numIdx],
@@ -125,36 +143,45 @@ namespace RcclUnitTesting
sendRank);
if (recvRank == 0)
{
testBed.AllocateMem(inPlace, useManagedMem, groupCallId, 0, sendRank, userRegistered);
testBed.PrepareData(groupCallId, 0, sendRank);
}
if (recvRank != sendRank)
{
if (testBed.ev.showNames) // Show test names
INFO("%s Datatype: %s SendReceive test Rank %d -> Rank %d for %d Elements\n",
isMultiProcess ? "MP" : "SP",
ncclDataTypeNames[dataTypes[dataIdx]],
sendRank,
recvRank,
numElements[numIdx]);
options.root = sendRank;
testBed.SetCollectiveArgs(ncclCollRecv,
testBed.SetCollectiveArgs(ncclCollSend,
dataTypes[dataIdx],
numElements[numIdx],
numElements[numIdx],
options,
0,
groupCallId,
recvRank);
testBed.AllocateMem(inPlace, useManagedMem, groupCallId, 0, recvRank, userRegistered);
testBed.PrepareData(groupCallId, 0, recvRank);
testBed.ExecuteCollectives({sendRank, recvRank});
testBed.ValidateResults(isCorrect, groupCallId, 0, recvRank);
testBed.DeallocateMem(groupCallId, 0, recvRank);
!groupCallId,
sendRank);
testBed.AllocateMem(inPlace, useManagedMem, 0, 0, sendRank, userRegistered);
testBed.PrepareData(0, 0, sendRank);
testBed.AllocateMem(inPlace, useManagedMem, 1, 0, sendRank, userRegistered);
testBed.PrepareData(1, 0, sendRank);
}
if (testBed.ev.showNames) // Show test names
INFO("%s Datatype: %s SendReceive test Rank %d -> Rank %d for %d Elements\n",
isMultiProcess ? "MP" : "SP",
ncclDataTypeNames[dataTypes[dataIdx]],
sendRank,
recvRank,
numElements[numIdx]);
options.root = sendRank;
testBed.SetCollectiveArgs(ncclCollRecv,
dataTypes[dataIdx],
numElements[numIdx],
numElements[numIdx],
options,
recvId,
groupCallId,
recvRank);
testBed.AllocateMem(inPlace, useManagedMem, groupCallId, recvId, recvRank, userRegistered);
testBed.PrepareData(groupCallId, recvId, recvRank);
testBed.ExecuteCollectives({sendRank, recvRank}, groupCallId);
testBed.ValidateResults(isCorrect, groupCallId, recvId, recvRank);
testBed.DeallocateMem(groupCallId, recvId, recvRank);
}
testBed.DeallocateMem(groupCallId, 0, sendRank);
testBed.DeallocateMem(0, 0, sendRank);
testBed.DeallocateMem(1, 0, sendRank);
}
testBed.DestroyComms();
}