AI/rocm-systems
2
0
Derivar 0
Código Questões Pedidos de integração Operações Pacotes Planeamentos Lançamentos Wiki Trabalho

Merge commit '3d4813d99196bb349eccd50a925e2addc8f1622c' into develop

Ver a proveniência
Este cometimento está contido em:
Ameya Keshava Mallya
2026-01-21 20:28:14 +00:00
ascendente 12ab8df3bc 3d4813d991
cometimento 8d996cc05f
295 ficheiros modificados com 27704 adições e 3800 eliminações
Descarregar ficheiro patch Descarregar ficheiro diff Expandir todos os ficheiros Colapsar todos os ficheiros
projects/rccl/test/AllToAllTests.cpp
+10 -10
Ver ficheiro
@@ -10,12 +10,12 @@
namespace RcclUnitTesting
{
TEST(AllToAll, OutOfPlace)
TEST(AlltoAll, OutOfPlace)
{
TestBed testBed;
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollAllToAll};
std::vector<ncclFunc_t> const funcTypes = {ncclCollAlltoAll};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat16, ncclFloat32};
std::vector<ncclRedOp_t> const redOps = {ncclSum};
std::vector<int> const roots = {0};
@@ -29,12 +29,12 @@ namespace RcclUnitTesting
testBed.Finalize();
}
TEST(AllToAll, OutOfPlaceGraph)
TEST(AlltoAll, OutOfPlaceGraph)
{
TestBed testBed;
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollAllToAll};
std::vector<ncclFunc_t> const funcTypes = {ncclCollAlltoAll};
std::vector<ncclDataType_t> const dataTypes = {ncclFloat64, ncclBfloat16, ncclFloat8e4m3, ncclFloat8e5m2};
std::vector<ncclRedOp_t> const redOps = {ncclSum};
std::vector<int> const roots = {0};
@@ -48,12 +48,12 @@ namespace RcclUnitTesting
testBed.Finalize();
}
TEST(AllToAll, ManagedMem)
TEST(AlltoAll, ManagedMem)
{
TestBed testBed;
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollAllToAll};
std::vector<ncclFunc_t> const funcTypes = {ncclCollAlltoAll};
std::vector<ncclDataType_t> const dataTypes = {ncclUint8};
std::vector<ncclRedOp_t> const redOps = {ncclSum};
std::vector<int> const roots = {0};
@@ -67,12 +67,12 @@ namespace RcclUnitTesting
testBed.Finalize();
}
TEST(AllToAll, ManagedMemGraph)
TEST(AlltoAll, ManagedMemGraph)
{
TestBed testBed;
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollAllToAll};
std::vector<ncclFunc_t> const funcTypes = {ncclCollAlltoAll};
std::vector<ncclDataType_t> const dataTypes = {ncclUint32, ncclUint64};
std::vector<ncclRedOp_t> const redOps = {ncclSum};
std::vector<int> const roots = {0};
@@ -86,13 +86,13 @@ namespace RcclUnitTesting
testBed.Finalize();
}
TEST(AllToAll, Channels)
TEST(AlltoAll, Channels)
{
TestBed testBed;
if(testBed.ev.maxGpus >= 8) {
if(testBed.ev.isGfx94) {
// Configuration
std::vector<ncclFunc_t> const funcTypes = {ncclCollAllToAll};
std::vector<ncclFunc_t> const funcTypes = {ncclCollAlltoAll};
std::vector<ncclDataType_t> const dataTypes = {ncclBfloat16};
std::vector<ncclRedOp_t> const redOps = {ncclSum};
std::vector<int> const roots = {0};
projects/rccl/test/AllToAllVTests.cpp
+8 -8
Ver ficheiro
@@ -74,7 +74,7 @@ namespace RcclUnitTesting
}
}
TEST(AllToAllv, OutOfPlace)
TEST(AlltoAllv, OutOfPlace)
{
TestBed testBed;
@@ -100,7 +100,7 @@ namespace RcclUnitTesting
const std::vector<int>& gpuPriorityOrder = testBed.ev.GetGpuPriorityOrder();
testBed.InitComms(TestBed::GetDeviceIdsList(numProcesses, totalRanks, gpuPriorityOrder));
// Prepare AllToAllV options
// Prepare AlltoAllV options
std::vector<size_t> numInputElements;
std::vector<size_t> numOutputElements;
PrepareCounts(totalRanks, 256, options, numInputElements, numOutputElements, 40);
@@ -110,14 +110,14 @@ namespace RcclUnitTesting
if (testBed.ev.showNames)
{
std::string name = testBed.GetTestCaseName(totalRanks, isMultiProcess,
ncclCollAllToAllv, dataTypes[dataIdx],
ncclCollAlltoAllv, dataTypes[dataIdx],
ncclSum, -1, inPlace, useManagedMem, useHipGraph);
INFO("%s\n", name.c_str());
}
for (int rank = 0; rank < totalRanks; ++rank)
{
testBed.SetCollectiveArgs(ncclCollAllToAllv,
testBed.SetCollectiveArgs(ncclCollAlltoAllv,
dataTypes[dataIdx],
numInputElements[rank],
numOutputElements[rank],
@@ -138,7 +138,7 @@ namespace RcclUnitTesting
}
TEST(AllToAllv, OutOfPlaceGraph)
TEST(AlltoAllv, OutOfPlaceGraph)
{
TestBed testBed;
@@ -164,7 +164,7 @@ namespace RcclUnitTesting
const std::vector<int>& gpuPriorityOrder = testBed.ev.GetGpuPriorityOrder();
testBed.InitComms(TestBed::GetDeviceIdsList(numProcesses, totalRanks, gpuPriorityOrder));
// Prepare AllToAllV options
// Prepare AlltoAllV options
std::vector<size_t> numInputElements;
std::vector<size_t> numOutputElements;
PrepareCounts(totalRanks, 256, options, numInputElements, numOutputElements, 60);
@@ -174,14 +174,14 @@ namespace RcclUnitTesting
if (testBed.ev.showNames)
{
std::string name = testBed.GetTestCaseName(totalRanks, isMultiProcess,
ncclCollAllToAllv, dataTypes[dataIdx],
ncclCollAlltoAllv, dataTypes[dataIdx],
ncclSum, -1, inPlace, useManagedMem, useHipGraph);
INFO("%s\n", name.c_str());
}
for (int rank = 0; rank < totalRanks; ++rank)
{
testBed.SetCollectiveArgs(ncclCollAllToAllv,
testBed.SetCollectiveArgs(ncclCollAlltoAllv,
dataTypes[dataIdx],
numInputElements[rank],
numOutputElements[rank],
projects/rccl/test/AltRsmiTests.cpp
+1276
Ver ficheiro
A apresentação das diferenças no ficheiro foi suprimida por ser demasiado grande Carregar diff
projects/rccl/test/CMakeLists.txt
+21
Ver ficheiro
@@ -153,6 +153,7 @@ if(BUILD_TESTS)
NonBlockingTests.cpp
ReduceScatterTests.cpp
ReduceTests.cpp
RegisterTests.cpp
ScatterTests.cpp
SendRecvTests.cpp
StandaloneTests.cpp
@@ -163,6 +164,7 @@ if(BUILD_TESTS)
common/EnvVars.cpp
common/PrepDataFuncs.cpp
common/PtrUnion.cpp
common/ProcessIsolatedTestRunner.cpp
common/TestBed.cpp
common/TestBedChild.cpp
common/StandaloneUtils.cpp
@@ -236,6 +238,25 @@ if(BUILD_TESTS)
list(APPEND RCCL_TEST_EXECUTABLES rccl-UnitTestsMPI)
endif()
# rccl-UnitTestsAltRsmi: Uses TEST BUILD alt_rsmi.cc (ARSMI_TEST_BUILD)
# This separate executable compiles alt_rsmi.cc with ARSMI_TEST_BUILD,
# enabling external linkage of internal variables so that
# tests can access and manipulate them for testing.
list(APPEND RCCL_TEST_EXECUTABLES rccl-UnitTestsAltRsmi)
set(TEST_ALTRSMI_SOURCE_FILES
AltRsmiTests.cpp
../src/misc/alt_rsmi.cc
common/main_fixtures.cpp
common/EnvVars.cpp
common/ProcessIsolatedTestRunner.cpp
)
add_executable(rccl-UnitTestsAltRsmi ${TEST_ALTRSMI_SOURCE_FILES})
# Define ARSMI_TEST_BUILD specifically for rccl-UnitTestsAltRsmi
target_compile_definitions(rccl-UnitTestsAltRsmi PRIVATE ARSMI_TEST_BUILD)
endif()
foreach(test_executable IN LISTS RCCL_TEST_EXECUTABLES)
projects/rccl/test/GroupCallTests.cpp
+2 -2
Ver ficheiro
@@ -81,7 +81,7 @@ namespace RcclUnitTesting
ncclCollAllReduce,
ncclCollGather,
ncclCollScatter,
ncclCollAllToAll};
ncclCollAlltoAll};
int const numCollPerGroup = funcTypes.size();
int const numElements = 1048576;
bool const inPlace = false;
@@ -251,7 +251,7 @@ namespace RcclUnitTesting
// Configuration
std::vector<std::vector<ncclFunc_t>> const groupCalls = {{ncclCollAllReduce, ncclCollAllGather},
{ncclCollAllToAll, ncclCollGather},
{ncclCollAlltoAll, ncclCollGather},
{ncclCollBroadcast, ncclCollReduceScatter}};
std::vector<std::vector<int>> const numElements = {{1250, 1048576}, {384, 384 * 1024}, {1048576, 127}};
std::vector<ncclDataType_t> const testDataTypes = {ncclFloat16, ncclFloat32, ncclBfloat16};
projects/rccl/test/NetSocketTests.cpp
+27 -17
Ver ficheiro
@@ -143,7 +143,12 @@ private:
protected:
void SetUp() override {
ncclResult_t result = ncclNetSocket.init(nullptr, nullptr);
void* ctx = nullptr;
uint64_t commId = 0;
ncclNetCommConfig_t config = {};
ncclDebugLogger_t logFunction = nullptr;
ncclProfilerCallback_t profFunction = nullptr;
ncclResult_t result = ncclNetSocket.init(&ctx, commId, &config, logFunction, profFunction);
ASSERT_EQ(result, ncclSuccess) << "Failed to initialize ncclNetSocket. "
<< "Error code: " << result
<< ". Ensure RCCL networking is properly configured.";
@@ -256,8 +261,9 @@ protected:
// Increased attempts and longer total timeout for reliability
for (int attempt = 0; attempt < 100 && !shouldStop.load(); attempt++) {
ncclResult_t connectResult = ncclNetSocket.connect(
0, &config, handle, &tempSendComm, &sendDevComm);
void* ctx = nullptr;
int dev = 0;
ncclResult_t connectResult = ncclNetSocket.connect(ctx, dev, handle, &tempSendComm, &sendDevComm);
if (connectResult == ncclSuccess && tempSendComm != nullptr) {
sendGuard.reset(tempSendComm);
connectCompleted = true;
@@ -702,7 +708,7 @@ protected:
char handle[NCCL_NET_HANDLE_MAXSIZE];
void *listenComm = nullptr;
ncclResult_t result = ncclNetSocket.listen(0, handle, &listenComm);
ncclResult_t result = ncclNetSocket.listen(nullptr, 0, handle, &listenComm);
ASSERT_EQ(result, ncclSuccess) << "Failed to establish listening socket for test execution. "
<< "ncclNetSocket.listen() returned error code: " << result
<< ". Verify network device availability and port accessibility.";
@@ -791,7 +797,9 @@ TEST_F(NetSocketTests, TestConcurrentOperationsTaskCreationDefault) {
char handle[NCCL_NET_HANDLE_MAXSIZE];
void *listenComm = nullptr;
ncclResult_t result = ncclNetSocket.listen(0, handle, &listenComm);
void* ctx = nullptr;
int dev = 0;
ncclResult_t result = ncclNetSocket.listen(ctx, dev, handle, &listenComm);
ASSERT_EQ(result, ncclSuccess) << "Failed to establish listening socket for test execution. "
<< "ncclNetSocket.listen() returned error code: " << result
<< ". Verify network device availability and port accessibility.";
@@ -895,7 +903,9 @@ TEST_F(NetSocketTests, TestInvalidDeviceIndexListen) {
void *listenComm = nullptr;
// Test with negative device index
ncclResult_t result = ncclNetSocket.listen(-1, handle, &listenComm);
void* ctx = nullptr;
int dev = -1;
ncclResult_t result = ncclNetSocket.listen(ctx, dev, handle, &listenComm);
INFO(NCCL_LOG_INFO, "Listen with dev=-1 returned: %d", result);
EXPECT_EQ(result, ncclInternalError)
<< "Listen should fail with negative device index. "
@@ -904,7 +914,7 @@ TEST_F(NetSocketTests, TestInvalidDeviceIndexListen) {
// Test with device index greater than available devices
int invalidDev = ndev + 10;
result = ncclNetSocket.listen(invalidDev, handle, &listenComm);
result = ncclNetSocket.listen(ctx, invalidDev, handle, &listenComm);
INFO(NCCL_LOG_INFO, "Listen with dev=%d (> ndev=%d) returned: %d", invalidDev,
ndev, result);
EXPECT_EQ(result, ncclInternalError)
@@ -926,8 +936,9 @@ TEST_F(NetSocketTests, TestInvalidDeviceIndexConnect) {
ncclNetDeviceHandle_t *sendDevComm = nullptr;
// Test with negative device index
ncclResult_t result =
ncclNetSocket.connect(-1, &config, handle, &sendComm, &sendDevComm);
void* ctx = nullptr;
int dev = -1;
ncclResult_t result = ncclNetSocket.connect(ctx, dev, handle, &sendComm, &sendDevComm);
INFO(NCCL_LOG_INFO, "Connect with dev=-1 returned: %d", result);
EXPECT_EQ(result, ncclInternalError)
<< "Connect should fail with negative device index. "
@@ -936,8 +947,7 @@ TEST_F(NetSocketTests, TestInvalidDeviceIndexConnect) {
// Test with device index greater than available devices
int invalidDev = ndev + 10;
result = ncclNetSocket.connect(invalidDev, &config, handle, &sendComm,
&sendDevComm);
result = ncclNetSocket.connect(ctx, invalidDev, handle, &sendComm, &sendDevComm);
INFO(NCCL_LOG_INFO, "Connect with dev=%d (> ndev=%d) returned: %d",
invalidDev, ndev, result);
EXPECT_EQ(result, ncclInternalError)
@@ -973,7 +983,7 @@ TEST_F(NetSocketTests, TestInvalidArraySizeIrecv) {
// Setup a dummy communicator first
char handle[NCCL_NET_HANDLE_MAXSIZE];
void *listenComm = nullptr;
ncclResult_t result = ncclNetSocket.listen(0, handle, &listenComm);
ncclResult_t result = ncclNetSocket.listen(nullptr, 0, handle, &listenComm);
if (result == ncclSuccess && listenComm) {
void *sendComm = nullptr;
@@ -1039,7 +1049,7 @@ TEST_F(NetSocketTests, TestNonHostMemoryRegMr) {
// Setup a dummy communicator first
char handle[NCCL_NET_HANDLE_MAXSIZE];
void *listenComm = nullptr;
ncclResult_t result = ncclNetSocket.listen(0, handle, &listenComm);
ncclResult_t result = ncclNetSocket.listen(nullptr, 0, handle, &listenComm);
if (result == ncclSuccess && listenComm) {
void *sendComm = nullptr;
@@ -1189,7 +1199,7 @@ TEST_F(NetSocketTests, TestExcessiveThreadConfig) {
// Initialize to trigger the warning logic
char handle[NCCL_NET_HANDLE_MAXSIZE];
void *listenComm = nullptr;
ncclResult_t result = ncclNetSocket.listen(0, handle, &listenComm);
ncclResult_t result = ncclNetSocket.listen(nullptr, 0, handle, &listenComm);
if (result == ncclSuccess && listenComm) {
// The implementation should have limited the threads to
@@ -1308,7 +1318,7 @@ TEST_F(NetSocketTests, TestExcessiveSocketConfig) {
// Initialize to trigger the warning logic
char handle[NCCL_NET_HANDLE_MAXSIZE];
void *listenComm = nullptr;
ncclResult_t result = ncclNetSocket.listen(0, handle, &listenComm);
ncclResult_t result = ncclNetSocket.listen(nullptr, 0, handle, &listenComm);
if (result == ncclSuccess && listenComm) {
// The implementation should have limited the sockets to
@@ -1337,7 +1347,7 @@ TEST_F(NetSocketTests, TestRequestAllocationFailure) {
// Setup communication
char handle[NCCL_NET_HANDLE_MAXSIZE];
void *listenComm = nullptr;
ncclResult_t result = ncclNetSocket.listen(0, handle, &listenComm);
ncclResult_t result = ncclNetSocket.listen(nullptr, 0, handle, &listenComm);
if (result == ncclSuccess && listenComm) {
void *sendComm = nullptr;
@@ -1441,7 +1451,7 @@ TEST_F(NetSocketTests, TestMessageSizeMismatch) {
char handle[NCCL_NET_HANDLE_MAXSIZE];
void *listenComm = nullptr;
ncclResult_t result = ncclNetSocket.listen(0, handle, &listenComm);
ncclResult_t result = ncclNetSocket.listen(nullptr, 0, handle, &listenComm);
if (result == ncclSuccess && listenComm) {
void *sendComm = nullptr;
projects/rccl/test/RegisterTests.cpp
+247
Ver ficheiro
@@ -0,0 +1,247 @@
/*************************************************************************
* Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
*
* See LICENSE.txt for license information
************************************************************************/
#include <gtest/gtest.h>
#include <rccl/rccl.h>
#include <cstdlib>
#include <cstdio>
#include "common/ErrCode.hpp"
#include "common/ProcessIsolatedTestRunner.hpp"
#include "StandaloneUtils.hpp"
namespace RcclUnitTesting
{
// Helper to check GPU availability
static bool hasGpuAvailable() {
int numDevices = 0;
hipError_t err = hipGetDeviceCount(&numDevices);
return (err == hipSuccess && numDevices >= 1);
}
// Macro to skip test if no GPU is available
#define SKIP_IF_NO_GPU() \
do { \
if (!hasGpuAvailable()) { \
GTEST_SKIP() << "This test requires at least 1 GPU device."; \
return; \
} \
} while(0)
// Helper to initialize a single-rank communicator
static ncclResult_t initSingleRankComm(ncclComm_t* comm) {
ncclUniqueId id;
ncclResult_t res = ncclGetUniqueId(&id);
if (res != ncclSuccess) return res;
return ncclCommInitRank(comm, 1, id, 0);
}
//==============================================================================
// Test implementation functions - parameterized by registration expectation
//==============================================================================
/**
* @brief Test basic register/deregister of a single buffer
* @param expectNonNull If true, expect non-NULL handle (registration enabled)
*/
static void testCommRegisterDeregister(bool expectNonNull) {
SKIP_IF_NO_GPU();
HIPCALL(hipSetDevice(0));
ncclComm_t comm;
NCCLCHECK(initSingleRankComm(&comm));
// Create buffer on device
const size_t bufferSize = 1024 * 1024; // 1 MB
void* deviceBuffer = nullptr;
HIPCALL(hipMalloc(&deviceBuffer, bufferSize));
ASSERT_NE(deviceBuffer, nullptr) << "Failed to allocate device buffer";
// Register buffer with ncclCommRegister
void* regHandle = nullptr;
NCCLCHECK(ncclCommRegister(comm, deviceBuffer, bufferSize, &regHandle));
// Verify handle based on expected behavior
if (expectNonNull) {
EXPECT_NE(regHandle, nullptr)
<< "Buffer registration failed: regHandle is NULL even though NCCL_LOCAL_REGISTER=1";
} else {
EXPECT_EQ(regHandle, nullptr)
<< "Expected NULL handle when NCCL_LOCAL_REGISTER is disabled";
}
// Deregister and clean up
NCCLCHECK(ncclCommDeregister(comm, regHandle));
HIPCALL(hipFree(deviceBuffer));
NCCLCHECK(ncclCommDestroy(comm));
}
/**
* @brief Test registering multiple buffers simultaneously
* @param expectNonNull If true, expect non-NULL handles and verify uniqueness
*/
static void testMultipleBufferRegistration(bool expectNonNull) {
SKIP_IF_NO_GPU();
HIPCALL(hipSetDevice(0));
ncclComm_t comm;
NCCLCHECK(initSingleRankComm(&comm));
// Create and register multiple buffers
const int numBuffers = 4;
const size_t bufferSize = 64 * 1024; // 64 KB each
void* deviceBuffers[numBuffers] = {nullptr};
void* regHandles[numBuffers] = {nullptr};
for (int i = 0; i < numBuffers; i++) {
HIPCALL(hipMalloc(&deviceBuffers[i], bufferSize));
ASSERT_NE(deviceBuffers[i], nullptr) << "Failed to allocate buffer " << i;
NCCLCHECK(ncclCommRegister(comm, deviceBuffers[i], bufferSize, &regHandles[i]));
if (expectNonNull) {
EXPECT_NE(regHandles[i], nullptr) << "Registration failed for buffer " << i;
} else {
EXPECT_EQ(regHandles[i], nullptr) << "Expected NULL handle for buffer " << i;
}
}
// Verify all handles are unique (only when registration is enabled)
if (expectNonNull) {
for (int i = 0; i < numBuffers; i++) {
for (int j = i + 1; j < numBuffers; j++) {
if (regHandles[i] != nullptr && regHandles[j] != nullptr) {
EXPECT_NE(regHandles[i], regHandles[j])
<< "Buffers " << i << " and " << j << " have the same registration handle";
}
}
}
}
// Deregister and clean up
for (int i = 0; i < numBuffers; i++) {
NCCLCHECK(ncclCommDeregister(comm, regHandles[i]));
HIPCALL(hipFree(deviceBuffers[i]));
}
NCCLCHECK(ncclCommDestroy(comm));
}
/**
* @brief Test registering buffers of various sizes
* @param expectNonNull If true, expect non-NULL handles for all sizes
*/
static void testVariableSizeBuffers(bool expectNonNull) {
SKIP_IF_NO_GPU();
HIPCALL(hipSetDevice(0));
ncclComm_t comm;
NCCLCHECK(initSingleRankComm(&comm));
// Test various buffer sizes: 4KB, 64KB, 1MB, 4MB
const size_t sizes[] = {4096, 64 * 1024, 1024 * 1024, 4 * 1024 * 1024};
const int numSizes = sizeof(sizes) / sizeof(sizes[0]);
for (int i = 0; i < numSizes; i++) {
void* deviceBuffer = nullptr;
void* regHandle = nullptr;
HIPCALL(hipMalloc(&deviceBuffer, sizes[i]));
ASSERT_NE(deviceBuffer, nullptr) << "Failed to allocate buffer of size " << sizes[i];
NCCLCHECK(ncclCommRegister(comm, deviceBuffer, sizes[i], &regHandle));
if (expectNonNull) {
EXPECT_NE(regHandle, nullptr)
<< "Registration failed for buffer size " << sizes[i] << " bytes";
} else {
EXPECT_EQ(regHandle, nullptr)
<< "Expected NULL handle for buffer size " << sizes[i] << " bytes";
}
NCCLCHECK(ncclCommDeregister(comm, regHandle));
HIPCALL(hipFree(deviceBuffer));
}
NCCLCHECK(ncclCommDestroy(comm));
}
/**
* @brief Test deregistering NULL handle (should succeed as no-op)
*/
static void testDeregisterNullHandle() {
SKIP_IF_NO_GPU();
HIPCALL(hipSetDevice(0));
ncclComm_t comm;
NCCLCHECK(initSingleRankComm(&comm));
// Deregister NULL handle - should be a no-op
NCCLCHECK(ncclCommDeregister(comm, nullptr));
NCCLCHECK(ncclCommDestroy(comm));
}
//==============================================================================
// Test configuration helpers
//==============================================================================
// Environment configuration for disabled registration (default)
static ProcessIsolatedTestRunner::TestConfig
makeDisabledConfig(const std::string& name, std::function<void()> testFn) {
return ProcessIsolatedTestRunner::TestConfig(name, testFn)
.clearVariable("NCCL_LOCAL_REGISTER");
}
// Environment configuration for enabled registration
static ProcessIsolatedTestRunner::TestConfig
makeEnabledConfig(const std::string& name, std::function<void()> testFn) {
return ProcessIsolatedTestRunner::TestConfig(name, testFn)
.withEnvironment({{"NCCL_LOCAL_REGISTER", "1"}});
}
/**
* @brief Test ncclCommRegister and ncclCommDeregister APIs with process isolation
*
* This test suite verifies that:
* 1. A device buffer can be registered with ncclCommRegister (API returns success)
* 2. When NCCL_LOCAL_REGISTER=1, the registration returns a valid (non-NULL) handle
* 3. When NCCL_LOCAL_REGISTER is not set, NULL handle is expected (default behavior)
* 4. The buffer can be deregistered with ncclCommDeregister
*
* Note: NCCL_LOCAL_REGISTER defaults to 0 (disabled) in RCCL.
*/
TEST(Register, ProcessIsolatedRegisterTests)
{
RUN_ISOLATED_TESTS(
// CommRegisterDeregister tests
makeDisabledConfig("CommRegisterDeregister_Disabled",
[]() { testCommRegisterDeregister(false); }),
makeEnabledConfig("CommRegisterDeregister_Enabled",
[]() { testCommRegisterDeregister(true); }),
// MultipleBufferRegistration tests
makeDisabledConfig("MultipleBufferRegistration_Disabled",
[]() { testMultipleBufferRegistration(false); }),
makeEnabledConfig("MultipleBufferRegistration_Enabled",
[]() { testMultipleBufferRegistration(true); }),
// VariableSizeBuffers tests
makeDisabledConfig("VariableSizeBuffers_Disabled",
[]() { testVariableSizeBuffers(false); }),
makeEnabledConfig("VariableSizeBuffers_Enabled",
[]() { testVariableSizeBuffers(true); }),
// DeregisterNullHandle test (no enable/disable variants needed)
ProcessIsolatedTestRunner::TestConfig("DeregisterNullHandle", testDeregisterNullHandle)
);
}
} // namespace RcclUnitTesting
projects/rccl/test/common/CollectiveArgs.cpp
+3 -3
Ver ficheiro
@@ -192,8 +192,8 @@ namespace RcclUnitTesting
case ncclCollAllReduce: ss << "ncclAllReduce"; break;
case ncclCollGather: ss << "ncclGather"; break;
case ncclCollScatter: ss << "ncclScatter"; break;
case ncclCollAllToAll: ss << "ncclAllToAll"; break;
case ncclCollAllToAllv: ss << "ncclAllToAllv"; break;
case ncclCollAlltoAll: ss << "ncclAlltoAll"; break;
case ncclCollAlltoAllv: ss << "ncclAlltoAllv"; break;
case ncclCollSend: ss << "ncclSend"; break;
case ncclCollRecv: ss << "ncclRecv"; break;
default: ss << "[Unknown]"; break;
@@ -276,7 +276,7 @@ namespace RcclUnitTesting
*numInputElements = totalRanks * N;
*numOutputElements = N;
break;
case ncclCollAllToAll:
case ncclCollAlltoAll:
*numInputElements = totalRanks * N;
*numOutputElements = totalRanks * N;
break;
projects/rccl/test/common/CollectiveArgs.hpp
+4 -4
Ver ficheiro
@@ -20,8 +20,8 @@ namespace RcclUnitTesting
ncclCollAllReduce,
ncclCollGather,
ncclCollScatter,
ncclCollAllToAll,
ncclCollAllToAllv,
ncclCollAlltoAll,
ncclCollAlltoAllv,
ncclCollSend,
ncclCollRecv,
ncclNumFuncs
@@ -36,8 +36,8 @@ namespace RcclUnitTesting
"AllReduce",
"Gather",
"Scatter",
"AllToAll",
"AllToAllv",
"AlltoAll",
"AlltoAllv",
"Send",
"Recv"
};
projects/rccl/test/common/PrepDataFuncs.cpp
+2 -2
Ver ficheiro
@@ -22,8 +22,8 @@ namespace RcclUnitTesting
case ncclCollAllReduce: return DefaultPrepData_Reduce(collArgs, true);
case ncclCollGather: return DefaultPrepData_Gather(collArgs, false);
case ncclCollScatter: return DefaultPrepData_Scatter(collArgs);
case ncclCollAllToAll: return DefaultPrepData_AllToAll(collArgs);
case ncclCollAllToAllv: return DefaultPrepData_AllToAllv(collArgs);
case ncclCollAlltoAll: return DefaultPrepData_AllToAll(collArgs);
case ncclCollAlltoAllv: return DefaultPrepData_AllToAllv(collArgs);
case ncclCollSend: return DefaultPrepData_Send(collArgs);
case ncclCollRecv: return DefaultPrepData_Recv(collArgs);
default:
projects/rccl/test/common/ProcessIsolatedTestRunner.cpp
+10 -2
Ver ficheiro
@@ -402,6 +402,9 @@ bool ProcessIsolatedTestRunner::executeAllTests(const ExecutionOptions& options)
continue;
}
// Flush all output before fork to prevent child from inheriting unflushed buffers
fflush(NULL);
pid_t pid = fork();
if(pid == 0)
@@ -435,6 +438,10 @@ bool ProcessIsolatedTestRunner::executeAllTests(const ExecutionOptions& options)
{
INFO("Running isolated test '%s' (PID: %d)\n", testConfig.name.c_str(), pid);
}
// Flush parent's output before reading from child pipes to ensure proper ordering
fflush(stdout);
fflush(stderr);
int status;
CapturedOutput output = captureProcessOutput(stdout_fd, stderr_fd, pid, &status);
@@ -442,6 +449,9 @@ bool ProcessIsolatedTestRunner::executeAllTests(const ExecutionOptions& options)
auto duration
= std::chrono::duration_cast<std::chrono::milliseconds>(endTime - startTime);
// Display captured output BEFORE status messages for proper sequencing
displayCapturedOutput(output, testConfig.name);
TestResult testResult;
testResult.testName = testConfig.name;
testResult.processId = pid;
@@ -530,8 +540,6 @@ bool ProcessIsolatedTestRunner::executeAllTests(const ExecutionOptions& options)
testResult.errorMessage = "Failed to wait for process";
}
displayCapturedOutput(output, testConfig.name);
recordTestResult(testResult);
// Stop on first failure if requested
projects/rccl/test/common/TestBedChild.cpp
+6 -6
Ver ficheiro
@@ -647,18 +647,18 @@ namespace RcclUnitTesting
this->streams[groupId][localRank][collArg.streamIdx]),
"ncclScatter");
break;
case ncclCollAllToAll:
CHILD_NCCL_CALL_RANK(errCode, ncclAllToAll(
case ncclCollAlltoAll:
CHILD_NCCL_CALL_RANK(errCode, ncclAlltoAll(
collArg.inputGpu.ptr,
collArg.outputGpu.ptr,
collArg.numInputElements / collArg.totalRanks,
collArg.dataType,
this->comms[localRank],
this->streams[groupId][localRank][collArg.streamIdx]),
"ncclAllToAll");
"ncclAlltoAll");
break;
case ncclCollAllToAllv:
CHILD_NCCL_CALL_RANK(errCode, ncclAllToAllv(
case ncclCollAlltoAllv:
CHILD_NCCL_CALL_RANK(errCode, ncclAlltoAllv(
collArg.inputGpu.ptr,
collArg.options.sendcounts + (this->rankOffset + localRank)*this->totalRanks,
collArg.options.sdispls + (this->rankOffset + localRank)*this->totalRanks,
@@ -668,7 +668,7 @@ namespace RcclUnitTesting
collArg.dataType,
this->comms[localRank],
this->streams[groupId][localRank][collArg.streamIdx]),
"ncclAllToAllv");
"ncclAlltoAllv");
break;
case ncclCollSend:
CHILD_NCCL_CALL_RANK(errCode, ncclSend(
projects/rccl/test/proxy_trace/ProxyTraceUnitTests.cpp
+24 -37
Ver ficheiro
@@ -9,6 +9,7 @@
#include "proxy_trace/proxy_trace.h"
#include <cstdint>
#include <gtest/gtest.h>
#include <memory>
#include <unistd.h>
namespace RcclUnitTesting {
@@ -21,7 +22,7 @@ public:
int nSteps = 10;
void SetUp() override {
proxyState = new ncclProxyState();
facebook_rccl::proxyTraceInit(proxyState->proxyTrace, 0, commHash);
proxyState->proxyTrace = std::make_unique<facebook_rccl::ProxyTrace>(0);
EXPECT_NE(proxyState->proxyTrace, nullptr);
sub1 = new ncclProxySubArgs();
sub2 = new ncclProxySubArgs();
@@ -36,9 +37,12 @@ public:
delete proxyState;
}
void AddTraceOp(ncclProxySubArgs *sub, facebook_rccl::ProxyOpType opType) {
facebook_rccl::addNewProxyOp(proxyState->proxyTrace, sub->traceKey,
sub->traceInfo, opType, sub->channelId,
sub->nsteps, sub->nbytes, sub->peer);
proxyState->proxyTrace->addNewProxyOp(
sub->traceKey,
sub->traceInfo,
opType,
sub->channelId,
sub->nsteps, sub->nbytes, sub->peer);
}
};
@@ -49,16 +53,10 @@ TEST_F(ProxyTraceTestFixture, nonEmptySingleton) {
TEST_F(ProxyTraceTestFixture, addTraceOp) {
auto &tracer = proxyState->proxyTrace;
EXPECT_EQ(tracer->getOrCreateProxyOpId(sub1->traceKey.commHash,
sub1->traceKey.opCount),
0);
AddTraceOp(sub1, facebook_rccl::ProxyOpType::SEND);
EXPECT_EQ(sub1->traceKey.proxyOpId, 0);
AddTraceOp(sub2, facebook_rccl::ProxyOpType::RECV);
EXPECT_EQ(sub2->traceKey.proxyOpId, 1);
EXPECT_EQ(tracer->getOrCreateProxyOpId(sub1->traceKey.commHash,
sub1->traceKey.opCount),
2);
auto traceRecordPtr = tracer->getProxyTraceOpPtr(sub1->traceKey);
EXPECT_EQ(traceRecordPtr->opType, facebook_rccl::ProxyOpType::SEND);
}
@@ -73,9 +71,10 @@ TEST_F(ProxyTraceTestFixture, getMapSizeMB) {
EXPECT_GT(size2, size1);
// finish sub1
sub1->done = nSteps;
facebook_rccl::updateProxyOpCounter(tracer, sub1->traceKey,
facebook_rccl::ProxyCounterTypes::DONE,
sub1->done);
tracer->updateProxyOpCounter(
sub1->traceKey,
facebook_rccl::ProxyCounterTypes::DONE,
sub1->done);
// sub1 is now serialized and should be moved from activeOps to finishedOps
auto size3 = tracer->getMapSizeMB();
EXPECT_GT(size3, size1);
@@ -84,13 +83,14 @@ TEST_F(ProxyTraceTestFixture, getMapSizeMB) {
TEST_F(ProxyTraceTestFixture, updateTraceOp) {
auto &tracer = proxyState->proxyTrace;
AddTraceOp(sub1, facebook_rccl::ProxyOpType::SEND);
facebook_rccl::updateProxyOpCounter(
tracer, sub1->traceKey,
facebook_rccl::ProxyCounterTypes::KERNEL_COPY_READY, 1);
facebook_rccl::updateProxyOpCounter(
tracer, sub1->traceKey, facebook_rccl::ProxyCounterTypes::POSTED, 3);
facebook_rccl::updateProxyOpCounter(
tracer, sub1->traceKey, facebook_rccl::ProxyCounterTypes::TRANSMITTED, 2);
tracer->updateProxyOpCounter(
sub1->traceKey,
facebook_rccl::ProxyCounterTypes::KERNEL_COPY_READY,
1);
tracer->updateProxyOpCounter(
sub1->traceKey, facebook_rccl::ProxyCounterTypes::POSTED, 3);
tracer->updateProxyOpCounter(
sub1->traceKey, facebook_rccl::ProxyCounterTypes::TRANSMITTED, 2);
auto traceRecordPtr = tracer->getProxyTraceOpPtr(sub1->traceKey);
EXPECT_NE(traceRecordPtr, nullptr);
@@ -110,25 +110,12 @@ TEST_F(ProxyTraceTestFixture, updateTraceOp2) {
AddTraceOp(sub1, facebook_rccl::ProxyOpType::SEND);
int64_t rand = 123456789;
sub1->posted = rand;
facebook_rccl::updateProxyOpCounter(tracer, sub1->traceKey,
facebook_rccl::ProxyCounterTypes::POSTED,
sub1->posted);
tracer->updateProxyOpCounter(sub1->traceKey,
facebook_rccl::ProxyCounterTypes::POSTED,
sub1->posted);
auto traceRecordPtr = tracer->getProxyTraceOpPtr(sub1->traceKey);
EXPECT_EQ(traceRecordPtr->counters[facebook_rccl::ProxyCounterTypes::POSTED],
rand);
}
TEST_F(ProxyTraceTestFixture, memoryReclaim) {
auto &tracer = proxyState->proxyTrace;
tracer->resetAll();
AddTraceOp(sub1, facebook_rccl::ProxyOpType::SEND);
sub1->done = nSteps;
facebook_rccl::updateProxyOpCounter(tracer, sub1->traceKey,
facebook_rccl::ProxyCounterTypes::DONE,
sub1->done);
auto traceRecordPtr = tracer->getProxyTraceOpPtr(sub1->traceKey);
EXPECT_EQ(traceRecordPtr, nullptr);
EXPECT_GT(tracer->getMapSizeMB(), 0);
}
} // namespace RcclUnitTesting
} // namespace RcclUnitTesting