Merge commit '3d4813d99196bb349eccd50a925e2addc8f1622c' into develop
Этот коммит содержится в:
@@ -0,0 +1,49 @@
|
||||
# Compiled shared objects and binaries
|
||||
*.so
|
||||
*.o
|
||||
*.a
|
||||
*.out
|
||||
*.exe
|
||||
*.dll
|
||||
*.dylib
|
||||
*.bin
|
||||
*.elf
|
||||
|
||||
# Python cache
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
|
||||
# Build and test artifacts
|
||||
/build/
|
||||
*.log
|
||||
*.tmp
|
||||
*.swp
|
||||
|
||||
# Ignore all CSV files except scripts/sample_performance_data.csv
|
||||
*.csv
|
||||
!scripts/sample_performance_data.csv
|
||||
|
||||
# Ignore all .conf files except nccl_tuner.conf
|
||||
*.conf
|
||||
!nccl_tuner.conf
|
||||
|
||||
my_configs
|
||||
|
||||
# Ignore test binary
|
||||
test/test_plugin
|
||||
|
||||
# Editor/OS files
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Backup files
|
||||
*~
|
||||
*.bak
|
||||
|
||||
# Ignore by convention
|
||||
*.old
|
||||
*.orig
|
||||
|
||||
# Git
|
||||
.git/
|
||||
@@ -0,0 +1,26 @@
|
||||
# Find all C source files in current directory
|
||||
set(SRC_FILES
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/plugin.c
|
||||
)
|
||||
|
||||
# Create shared library
|
||||
add_library(nccl-tuner-example SHARED ${SRC_FILES})
|
||||
|
||||
# Set include directories
|
||||
target_include_directories(nccl-tuner-example PRIVATE
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/nccl
|
||||
)
|
||||
|
||||
# Set output name to match Makefile
|
||||
set_target_properties(nccl-tuner-example PROPERTIES
|
||||
OUTPUT_NAME "nccl-tuner-example"
|
||||
PREFIX "lib"
|
||||
POSITION_INDEPENDENT_CODE ON
|
||||
LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/test/unit/plugins
|
||||
)
|
||||
|
||||
# Add custom target for clean (equivalent to Makefile clean target)
|
||||
add_custom_target(clean-tuner-lib
|
||||
COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/libnccl-tuner-example.so
|
||||
COMMENT "Cleaning libnccl-tuner-example.so"
|
||||
)
|
||||
@@ -45,6 +45,40 @@ typedef enum {
|
||||
|
||||
#define NCCL_ALGO_PROTO_IGNORE -1.0
|
||||
|
||||
#define NCCL_HW_NVLINK 0
|
||||
#define NCCL_HW_PCI 1
|
||||
#define NCCL_HW_NET 2
|
||||
#define NCCL_NUM_HW_LINKS 3
|
||||
|
||||
#define NCCL_VOLTA_COMPCAP_IDX 0
|
||||
#define NCCL_AMPERE_COMPCAP_IDX 1
|
||||
#define NCCL_HOPPER_COMPCAP_IDX 2
|
||||
#define NCCL_BLACKWELL_COMPCAP_IDX 3
|
||||
#define NCCL_NUM_COMPCAPS 4
|
||||
|
||||
#define NCCL_TUNING_SCALE_1NODE 0
|
||||
#define NCCL_TUNING_SCALE_2NODES 1
|
||||
#define NCCL_TUNING_SCALE_4NODES 2
|
||||
#define NCCL_NUM_TUNING_SCALES 3
|
||||
|
||||
typedef struct {
|
||||
int nNvlDomains; // number of NVLink domains
|
||||
int minRanksPerNvlDomain; // minimum ranks across all NVLink domains
|
||||
int maxRanksPerNvlDomain; // maximum ranks across all NVLink domains
|
||||
} ncclNvlDomainInfo_v5_t;
|
||||
|
||||
typedef struct {
|
||||
double baseLatencies [NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
|
||||
double hwLatencies [NCCL_NUM_HW_LINKS][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
|
||||
|
||||
double llMaxBws [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES];
|
||||
double perChMaxRingLL128Bws [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES];
|
||||
double perChMaxTreeLL128Bws [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES];
|
||||
double perChMaxTreeBws [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES];
|
||||
|
||||
|
||||
} ncclTunerConstants_v5_t;
|
||||
|
||||
// API to be implemented by external tuner
|
||||
typedef struct {
|
||||
// Name of the tuner
|
||||
@@ -52,12 +86,17 @@ typedef struct {
|
||||
|
||||
// Initializes tuner states.
|
||||
// Inputs:
|
||||
// - commId: communicator identifier
|
||||
// - nRanks: number of ranks in current communicator. Each communicator initialize its own tuner.
|
||||
// - nNodes: number of nodes in current communicator.
|
||||
// - logFunction: a logFunction can be useful to integrate logging together with NCCL core.
|
||||
// - nvlDomainInfo: NVL domain information struct
|
||||
// Outputs:
|
||||
// - context: tuner context object
|
||||
ncclResult_t (*init)(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context);
|
||||
// Input/Output:
|
||||
// - constants: tuner constants
|
||||
ncclResult_t (*init)(void** ctx, uint64_t commId, size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction,
|
||||
ncclNvlDomainInfo_v5_t* nvlDomainInfo, ncclTunerConstants_v5_t* constants);
|
||||
|
||||
// Gets info (algo, protocol, number of ctas and threads) for a given collective.
|
||||
// Inputs:
|
||||
@@ -87,11 +126,13 @@ typedef struct {
|
||||
|
||||
// Terminates the plugin and cleans up any resources that the plugin allocated.
|
||||
// context: tuner context object
|
||||
ncclResult_t (*destroy)(void* context);
|
||||
} ncclTuner_v4_t;
|
||||
ncclResult_t (*finalize)(void* context);
|
||||
} ncclTuner_v5_t;
|
||||
|
||||
typedef ncclTuner_v4_t ncclTuner_t;
|
||||
typedef ncclTuner_v5_t ncclTuner_t;
|
||||
typedef ncclNvlDomainInfo_v5_t ncclNvlDomainInfo_t;
|
||||
typedef ncclTunerConstants_v5_t ncclTunerConstants_t;
|
||||
|
||||
#define NCCL_TUNER_PLUGIN_SYMBOL "ncclTunerPlugin_v4"
|
||||
#define NCCL_TUNER_PLUGIN_SYMBOL "ncclTunerPlugin_v5"
|
||||
|
||||
#endif
|
||||
|
||||
@@ -51,6 +51,7 @@ typedef struct {
|
||||
size_t nRanks;
|
||||
size_t nNodes;
|
||||
ncclDebugLogger_t logFunction;
|
||||
ncclNvlDomainInfo_v5_t nvlDomainInfo;
|
||||
} TunerContext;
|
||||
|
||||
// Parse collective type from string
|
||||
@@ -289,7 +290,25 @@ static ncclResult_t loadConfig(TunerContext* ctx, const char* filename) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
__hidden ncclResult_t pluginInit(size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction, void **context) {
|
||||
__hidden ncclResult_t pluginInit(void** context, uint64_t commId, size_t nRanks, size_t nNodes, ncclDebugLogger_t logFunction,
|
||||
ncclNvlDomainInfo_v5_t* nvlDomainInfo, ncclTunerConstants_v5_t* constants) {
|
||||
|
||||
if (NULL != constants) {
|
||||
// NCCL constants tuning
|
||||
// Tune NCCL's internal tuning model to improve base algo/proto selection.
|
||||
// Note: Example numbers are for reference only.
|
||||
// Actual numbers may vary depending on the hardware and network topology.
|
||||
// These numbers are not guaranteed to be optimal for all cases.
|
||||
// Limit the tree bandwidth to 15GB/s
|
||||
constants->perChMaxTreeBws[NCCL_BLACKWELL_COMPCAP_IDX][NCCL_TUNING_SCALE_4NODES] = 15.0;
|
||||
|
||||
// Limit the ring bandwidth to 20GB/s
|
||||
constants->perChMaxRingLL128Bws[NCCL_BLACKWELL_COMPCAP_IDX][NCCL_TUNING_SCALE_4NODES] = 20.0;
|
||||
|
||||
// Set NVLSTree base network latency to 24us
|
||||
constants->hwLatencies[NCCL_HW_NET][NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] = 24.0;
|
||||
}
|
||||
|
||||
TunerContext* ctx = (TunerContext*)malloc(sizeof(TunerContext));
|
||||
if (!ctx) return ncclSystemError;
|
||||
|
||||
@@ -299,10 +318,16 @@ __hidden ncclResult_t pluginInit(size_t nRanks, size_t nNodes, ncclDebugLogger_t
|
||||
ctx->nRanks = nRanks;
|
||||
ctx->nNodes = nNodes;
|
||||
ctx->logFunction = logFunction;
|
||||
if (nvlDomainInfo) {
|
||||
ctx->nvlDomainInfo = *nvlDomainInfo;
|
||||
} else {
|
||||
memset(&ctx->nvlDomainInfo, 0, sizeof(ncclNvlDomainInfo_v5_t));
|
||||
}
|
||||
|
||||
if (logFunction) {
|
||||
logFunction(NCCL_LOG_INFO, NCCL_TUNING, __FILE__, __LINE__,
|
||||
"TUNER/ExamplePlugin: Initializing tuner for %zu nodes, %zu ranks", nNodes, nRanks);
|
||||
"TUNER/ExamplePlugin: Initializing tuner for %zu nodes, %zu ranks, %d NVL domains",
|
||||
nNodes, nRanks, ctx->nvlDomainInfo.nNvlDomains);
|
||||
}
|
||||
|
||||
// Try to load config file from environment variable or default location
|
||||
@@ -435,7 +460,7 @@ __hidden ncclResult_t pluginGetCollInfo(void* context, ncclFunc_t collType, size
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
__hidden ncclResult_t pluginDestroy(void* context) {
|
||||
__hidden ncclResult_t pluginFinalize(void* context) {
|
||||
if (context) {
|
||||
TunerContext* ctx = (TunerContext*)context;
|
||||
if (ctx->configs) {
|
||||
@@ -446,11 +471,12 @@ __hidden ncclResult_t pluginDestroy(void* context) {
|
||||
return ncclSuccess;
|
||||
}
|
||||
|
||||
|
||||
#define PLUGIN_NAME "Example"
|
||||
|
||||
const ncclTuner_v4_t ncclTunerPlugin_v4 = {
|
||||
const ncclTuner_v5_t ncclTunerPlugin_v5 = {
|
||||
.name = PLUGIN_NAME,
|
||||
.init = pluginInit,
|
||||
.getCollInfo = pluginGetCollInfo,
|
||||
.destroy = pluginDestroy
|
||||
.finalize = pluginFinalize
|
||||
};
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# NCCL Tuner Configuration File (CSV Format)
|
||||
# Format: collective_type,min_bytes,max_bytes,algorithm,protocol,channels,nNodes,nRanks,numPipeOps,regBuff
|
||||
#
|
||||
# Collective types: broadcast, reduce, allgather, reducescatter, allreduce
|
||||
# Algorithms: tree, ring, collnet_direct, collnet_chain, nvls, nvls_tree, pat
|
||||
# Protocols: ll, ll128, simple
|
||||
# Channels: number of channels to use, or -1 to keep default
|
||||
# nNodes: number of nodes to match, or -1 for any number of nodes
|
||||
# nRanks: number of ranks to match, or -1 for any number of ranks
|
||||
# numPipeOps: number of pipeline operations to match, or -1 for any number (optional)
|
||||
# regBuff: whether user buffer can be registered (0=no, 1=yes, -1=any) (optional)
|
||||
#
|
||||
# Note: numPipeOps and regBuff parameters are optional - configurations without them will match any value
|
||||
#
|
||||
#AR 4PPN
|
||||
allreduce,33554432,4294967296,ring,simple,16,2,8,-1,-1
|
||||
allreduce,33554432,4294967296,ring,simple,16,4,16,-1,-1
|
||||
allreduce,67108864,4294967296,ring,simple,16,8,32,-1,-1
|
||||
#AR 2PPN
|
||||
allreduce,2097152,4294967296,ring,simple,4,2,4,-1,-1
|
||||
allreduce,16777216,4294967296,ring,simple,4,4,8,-1,-1
|
||||
allreduce,33554432,4294967296,ring,simple,4,8,16,-1,-1
|
||||
#AR 1PPN
|
||||
allreduce,134217728,4294967296,ring,simple,4,4,4,-1,-1
|
||||
allreduce,67108864,4294967296,ring,simple,4,8,8,-1,-1
|
||||
|
||||
|
||||
#AG 4PPN
|
||||
allgather,8388608,4294967296,ring,simple,16,2,8,-1,-1
|
||||
allgather,16777216,4294967296,ring,simple,16,4,16,-1,-1
|
||||
allgather,16777216,4294967296,ring,simple,16,8,32,-1,-1
|
||||
#AG 2PPN
|
||||
allgather,262144,4294967296,ring,simple,4,2,4,-1,-1
|
||||
allgather,16777216,4294967296,ring,simple,4,4,8,-1,-1
|
||||
allgather,33554432,4294967296,ring,simple,4,8,16,-1,-1
|
||||
#AG 1PPN
|
||||
allgather,262144,2097152,ring,simple,4,2,2,-1,-1
|
||||
allgather,262144,8388608,ring,simple,4,4,4,-1,-1
|
||||
allgather,67108864,4294967296,ring,simple,4,8,8,-1,-1
|
||||
|
||||
#RS 4PPN
|
||||
reducescatter,1048576,4294967296,ring,simple,16,2,8,-1,-1
|
||||
reducescatter,1048576,4294967296,ring,simple,16,4,16,-1,-1
|
||||
reducescatter,1048576,4294967296,ring,simple,16,8,32,-1,-1
|
||||
#RS 2PPN
|
||||
reducescatter,262144,33554432,ring,simple,4,2,4,-1,-1
|
||||
reducescatter,262144,4294967296,ring,simple,4,4,8,-1,-1
|
||||
reducescatter,262144,4294967296,ring,simple,4,8,16,-1,-1
|
||||
#RS 1PPN
|
||||
reducescatter,131072,262144,ring,simple,4,2,2,-1,-1
|
||||
reducescatter,1048576,2097152,ring,simple,4,2,2,-1,-1
|
||||
reducescatter,131072,4194304,ring,simple,4,4,4,-1,-1
|
||||
reducescatter,262144,8388608,ring,simple,4,8,8,-1,-1
|
||||
@@ -98,12 +98,12 @@ int test_plugin_init() {
|
||||
void* context = NULL;
|
||||
|
||||
// Test successful initialization
|
||||
ncclResult_t result = pluginInit(8, 2, mock_logger, &context);
|
||||
ncclResult_t result = pluginInit(&context, 0, 8, 2, mock_logger, NULL, NULL);
|
||||
TEST_ASSERT(result == ncclSuccess, "Plugin init should succeed");
|
||||
TEST_ASSERT(context != NULL, "Context should be allocated");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
TEST_PASS();
|
||||
}
|
||||
|
||||
@@ -123,11 +123,11 @@ int test_config_parsing_valid() {
|
||||
setenv("NCCL_TUNER_CONFIG_FILE", "test_valid.conf", 1);
|
||||
|
||||
void* context = NULL;
|
||||
ncclResult_t result = pluginInit(16, 2, mock_logger, &context);
|
||||
ncclResult_t result = pluginInit(&context, 0, 16, 2, mock_logger, NULL, NULL);
|
||||
TEST_ASSERT(result == ncclSuccess, "Plugin init with valid config should succeed");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink("test_valid.conf");
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
TEST_PASS();
|
||||
@@ -144,12 +144,12 @@ int test_config_parsing_invalid() {
|
||||
setenv("NCCL_TUNER_CONFIG_FILE", "test_invalid.conf", 1);
|
||||
|
||||
void* context = NULL;
|
||||
ncclResult_t result = pluginInit(8, 1, mock_logger, &context);
|
||||
ncclResult_t result = pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL);
|
||||
// Should still succeed but with no valid configs loaded
|
||||
TEST_ASSERT(result == ncclSuccess, "Plugin init should succeed even with invalid config");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink("test_invalid.conf");
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
TEST_PASS();
|
||||
@@ -165,7 +165,7 @@ int test_collective_matching() {
|
||||
setenv("NCCL_TUNER_CONFIG_FILE", "test_match.conf", 1);
|
||||
|
||||
void* context = NULL;
|
||||
pluginInit(8, 1, mock_logger, &context);
|
||||
pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL);
|
||||
|
||||
// Create mock cost table
|
||||
float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
|
||||
@@ -209,7 +209,7 @@ int test_collective_matching() {
|
||||
TEST_ASSERT(nChannels == 4, "Should set 4 channels");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink("test_match.conf");
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
TEST_PASS();
|
||||
@@ -226,7 +226,7 @@ int test_size_matching() {
|
||||
setenv("NCCL_TUNER_CONFIG_FILE", "test_size.conf", 1);
|
||||
|
||||
void* context = NULL;
|
||||
pluginInit(8, 1, mock_logger, &context);
|
||||
pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL);
|
||||
|
||||
float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
|
||||
float* cost_table_ptr[NCCL_NUM_ALGORITHMS];
|
||||
@@ -280,7 +280,7 @@ int test_size_matching() {
|
||||
TEST_ASSERT(nChannels == 8, "Large: Should set 8 channels");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink("test_size.conf");
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
TEST_PASS();
|
||||
@@ -298,7 +298,7 @@ int test_topology_matching() {
|
||||
|
||||
// Test with single node setup
|
||||
void* context1 = NULL;
|
||||
pluginInit(8, 1, mock_logger, &context1); // 8 ranks, 1 node
|
||||
pluginInit(&context1, 0, 8, 1, mock_logger, NULL, NULL); // 8 ranks, 1 node
|
||||
|
||||
float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
|
||||
float* cost_table_ptr[NCCL_NUM_ALGORITHMS];
|
||||
@@ -316,11 +316,11 @@ int test_topology_matching() {
|
||||
TEST_ASSERT(cost_table[NCCL_ALGO_TREE][NCCL_PROTO_SIMPLE] == 0.0, "Single node: Should match tree config");
|
||||
TEST_ASSERT(nChannels == 2, "Single node: Should set 2 channels");
|
||||
|
||||
pluginDestroy(context1);
|
||||
pluginFinalize(context1);
|
||||
|
||||
// Test with 4 nodes, 32 ranks setup
|
||||
void* context2 = NULL;
|
||||
pluginInit(32, 4, mock_logger, &context2); // 32 ranks, 4 nodes
|
||||
pluginInit(&context2, 0, 32, 4, mock_logger, NULL, NULL); // 32 ranks, 4 nodes
|
||||
|
||||
for (int i = 0; i < NCCL_NUM_ALGORITHMS; i++) {
|
||||
for (int j = 0; j < NCCL_NUM_PROTOCOLS; j++) {
|
||||
@@ -349,7 +349,7 @@ int test_default_channels() {
|
||||
setenv("NCCL_TUNER_CONFIG_FILE", "test_default.conf", 1);
|
||||
|
||||
void* context = NULL;
|
||||
pluginInit(8, 1, mock_logger, &context);
|
||||
pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL);
|
||||
|
||||
float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
|
||||
float* cost_table_ptr[NCCL_NUM_ALGORITHMS];
|
||||
@@ -369,7 +369,7 @@ int test_default_channels() {
|
||||
TEST_ASSERT(nChannels == 1, "Should keep default channels (1) when config has -1");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink("test_default.conf");
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
TEST_PASS();
|
||||
@@ -386,7 +386,7 @@ int test_regbuff_matching() {
|
||||
setenv("NCCL_TUNER_CONFIG_FILE", "test_regbuff.conf", 1);
|
||||
|
||||
void* context = NULL;
|
||||
pluginInit(8, 1, mock_logger, &context);
|
||||
pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL);
|
||||
|
||||
float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
|
||||
float* cost_table_ptr[NCCL_NUM_ALGORITHMS];
|
||||
@@ -437,7 +437,7 @@ int test_regbuff_matching() {
|
||||
TEST_ASSERT(nChannels == 8, "Any regBuff: Should set 8 channels");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink("test_regbuff.conf");
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
TEST_PASS();
|
||||
@@ -454,7 +454,7 @@ int test_pipeops_matching() {
|
||||
setenv("NCCL_TUNER_CONFIG_FILE", "test_pipeops.conf", 1);
|
||||
|
||||
void* context = NULL;
|
||||
pluginInit(8, 1, mock_logger, &context);
|
||||
pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL);
|
||||
|
||||
float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
|
||||
float* cost_table_ptr[NCCL_NUM_ALGORITHMS];
|
||||
@@ -504,7 +504,7 @@ int test_pipeops_matching() {
|
||||
TEST_ASSERT(nChannels == 8, "Any pipeOps: Should set 8 channels");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink("test_pipeops.conf");
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
TEST_PASS();
|
||||
@@ -519,7 +519,7 @@ int test_no_match_fallback() {
|
||||
setenv("NCCL_TUNER_CONFIG_FILE", "test_fallback.conf", 1);
|
||||
|
||||
void* context = NULL;
|
||||
pluginInit(8, 1, mock_logger, &context);
|
||||
pluginInit(&context, 0, 8, 1, mock_logger, NULL, NULL);
|
||||
|
||||
float cost_table[NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS];
|
||||
float* cost_table_ptr[NCCL_NUM_ALGORITHMS];
|
||||
@@ -543,7 +543,7 @@ int test_no_match_fallback() {
|
||||
TEST_ASSERT(nChannels == 1, "Should use default channels");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink("test_fallback.conf");
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
TEST_PASS();
|
||||
@@ -593,7 +593,7 @@ int test_large_config() {
|
||||
|
||||
// Initialize plugin with large config
|
||||
void* context = NULL;
|
||||
ncclResult_t result = pluginInit(16, 4, mock_logger, &context);
|
||||
ncclResult_t result = pluginInit(&context, 0, 16, 4, mock_logger, NULL, NULL);
|
||||
TEST_ASSERT(result == ncclSuccess, "Plugin init with large config should succeed");
|
||||
TEST_ASSERT(context != NULL, "Context should be allocated");
|
||||
|
||||
@@ -652,7 +652,7 @@ int test_large_config() {
|
||||
TEST_ASSERT(result == ncclSuccess, "GetCollInfo should work with large config set");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink(large_config_file);
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
|
||||
@@ -684,7 +684,7 @@ int test_very_large_config_stress() {
|
||||
|
||||
// Test initialization with stress config
|
||||
void* context = NULL;
|
||||
ncclResult_t result = pluginInit(8, 2, mock_logger, &context);
|
||||
ncclResult_t result = pluginInit(&context, 0, 8, 2, mock_logger, NULL, NULL);
|
||||
TEST_ASSERT(result == ncclSuccess, "Plugin should handle very large config files");
|
||||
|
||||
TunerContext* ctx = (TunerContext*)context;
|
||||
@@ -705,7 +705,7 @@ int test_very_large_config_stress() {
|
||||
}
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink(stress_config_file);
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
|
||||
@@ -726,7 +726,7 @@ int test_empty_config() {
|
||||
setenv("NCCL_TUNER_CONFIG_FILE", empty_config_file, 1);
|
||||
|
||||
void* context = NULL;
|
||||
ncclResult_t result = pluginInit(8, 2, mock_logger, &context);
|
||||
ncclResult_t result = pluginInit(&context, 0, 8, 2, mock_logger, NULL, NULL);
|
||||
TEST_ASSERT(result == ncclSuccess, "Plugin should handle empty config files");
|
||||
|
||||
TunerContext* ctx = (TunerContext*)context;
|
||||
@@ -751,13 +751,134 @@ int test_empty_config() {
|
||||
TEST_ASSERT(result == ncclSuccess, "GetCollInfo should work with empty config");
|
||||
|
||||
// Clean up
|
||||
pluginDestroy(context);
|
||||
pluginFinalize(context);
|
||||
unlink(empty_config_file);
|
||||
unsetenv("NCCL_TUNER_CONFIG_FILE");
|
||||
|
||||
TEST_PASS();
|
||||
}
|
||||
|
||||
// Test NVLink domain info handling
|
||||
int test_nvl_domain_info() {
|
||||
printf("Testing NVLink domain info handling...\n");
|
||||
|
||||
// Test NVLink domain structure with min/max ranks per domain
|
||||
ncclNvlDomainInfo_v5_t nvl_domain = {
|
||||
.nNvlDomains = 2, // 2 nodes = 2 domains
|
||||
.minRanksPerNvlDomain = 3, // minimum ranks across all domains (bottleneck)
|
||||
.maxRanksPerNvlDomain = 5 // maximum ranks across all domains (capacity)
|
||||
};
|
||||
|
||||
void* context = NULL;
|
||||
ncclResult_t result = pluginInit(&context, 0, 8, 2, mock_logger, &nvl_domain, NULL);
|
||||
TEST_ASSERT(result == ncclSuccess, "Plugin init with NVLink domains should succeed");
|
||||
|
||||
// Validate NVLD info structure
|
||||
TEST_ASSERT(nvl_domain.nNvlDomains == 2, "Should have 2 domains (nodes)");
|
||||
TEST_ASSERT(nvl_domain.minRanksPerNvlDomain == 3, "Should have minimum 3 ranks per domain");
|
||||
TEST_ASSERT(nvl_domain.maxRanksPerNvlDomain == 5, "Should have maximum 5 ranks per domain");
|
||||
|
||||
// Clean up
|
||||
pluginFinalize(context);
|
||||
printf("NVLink domain info test passed!\n");
|
||||
TEST_PASS();
|
||||
}
|
||||
|
||||
int test_tuner_constants() {
|
||||
// Initialize constants to -1.0 for testing purposes
|
||||
ncclTunerConstants_v5_t constants = {
|
||||
// Base latencies: [NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]
|
||||
.baseLatencies = {
|
||||
{-1.0, -1.0, -1.0}, // NCCL_ALGO_TREE: LL, LL128, Simple
|
||||
{-1.0, -1.0, -1.0}, // NCCL_ALGO_RING: LL, LL128, Simple
|
||||
{-1.0, -1.0, -1.0}, // NCCL_ALGO_COLLNET_DIRECT
|
||||
{-1.0, -1.0, -1.0}, // NCCL_ALGO_COLLNET_CHAIN
|
||||
{-1.0, -1.0, -1.0}, // NCCL_ALGO_NVLS
|
||||
{-1.0, -1.0, -1.0}, // NCCL_ALGO_NVLS_TREE
|
||||
{-1.0, -1.0, -1.0} // NCCL_ALGO_PAT
|
||||
},
|
||||
|
||||
// Hardware latencies: [NCCL_NUM_HW_LINKS][NCCL_NUM_ALGORITHMS][NCCL_NUM_PROTOCOLS]
|
||||
.hwLatencies = {
|
||||
// NCCL_HW_NVLINK
|
||||
{
|
||||
{-1.0, -1.0, -1.0}, // TREE
|
||||
{-1.0, -1.0, -1.0}, // RING
|
||||
{-1.0, -1.0, -1.0}, // COLLNET_DIRECT
|
||||
{-1.0, -1.0, -1.0}, // COLLNET_CHAIN
|
||||
{-1.0, -1.0, -1.0}, // NVLS
|
||||
{-1.0, -1.0, -1.0}, // NVLS_TREE
|
||||
{-1.0, -1.0, -1.0} // PAT
|
||||
},
|
||||
// NCCL_HW_PCI
|
||||
{
|
||||
{-1.0, -1.0, -1.0}, // TREE
|
||||
{-1.0, -1.0, -1.0}, // RING
|
||||
{-1.0, -1.0, -1.0}, // COLLNET_DIRECT
|
||||
{-1.0, -1.0, -1.0}, // COLLNET_CHAIN
|
||||
{-1.0, -1.0, -1.0}, // NVLS
|
||||
{-1.0, -1.0, -1.0}, // NVLS_TREE
|
||||
{-1.0, -1.0, -1.0} // PAT
|
||||
},
|
||||
// NCCL_HW_NET
|
||||
{
|
||||
{-1.0, -1.0, -1.0}, // TREE
|
||||
{-1.0, -1.0, -1.0}, // RING
|
||||
{-1.0, -1.0, -1.0}, // COLLNET_DIRECT
|
||||
{-1.0, -1.0, -1.0}, // COLLNET_CHAIN
|
||||
{-1.0, -1.0, -1.0}, // NVLS
|
||||
{-1.0, -1.0, -1.0}, // NVLS_TREE
|
||||
{-1.0, -1.0, -1.0} // PAT
|
||||
}
|
||||
},
|
||||
|
||||
// LL maximum bandwidths: [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES]
|
||||
.llMaxBws = {
|
||||
{-1.0, -1.0, -1.0}, // Volta: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0}, // Ampere: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0}, // Hopper: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0} // Blackwell: 1node, 2nodes, 4nodes
|
||||
},
|
||||
|
||||
// Per-channel maximum Ring LL128 bandwidths: [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES]
|
||||
.perChMaxRingLL128Bws = {
|
||||
{-1.0, -1.0, -1.0}, // Volta: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0}, // Ampere: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0}, // Hopper: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0} // Blackwell: 1node, 2nodes, 4nodes
|
||||
},
|
||||
|
||||
// Per-channel maximum Tree LL128 bandwidths: [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES]
|
||||
.perChMaxTreeLL128Bws = {
|
||||
{-1.0, -1.0, -1.0}, // Volta: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0}, // Ampere: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0}, // Hopper: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0} // Blackwell: 1node, 2nodes, 4nodes
|
||||
},
|
||||
|
||||
// Per-channel maximum Tree bandwidths: [NCCL_NUM_COMPCAPS][NCCL_NUM_TUNING_SCALES]
|
||||
.perChMaxTreeBws = {
|
||||
{-1.0, -1.0, -1.0}, // Volta: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0}, // Ampere: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0}, // Hopper: 1node, 2nodes, 4nodes
|
||||
{-1.0, -1.0, -1.0} // Blackwell: 1node, 2nodes, 4nodes
|
||||
}
|
||||
};
|
||||
|
||||
void* context = NULL;
|
||||
ncclResult_t result = pluginInit(&context, 0, 8, 2, mock_logger, NULL, &constants);
|
||||
TEST_ASSERT(result == ncclSuccess, "Plugin init with constants should succeed");
|
||||
|
||||
// Test that the constants were set correctly
|
||||
TEST_ASSERT(constants.perChMaxTreeBws[NCCL_BLACKWELL_COMPCAP_IDX][NCCL_TUNING_SCALE_4NODES] == 15.0, "Tree bandwidth should be 15GB/s");
|
||||
TEST_ASSERT(constants.perChMaxRingLL128Bws[NCCL_BLACKWELL_COMPCAP_IDX][NCCL_TUNING_SCALE_4NODES] == 20.0, "Ring bandwidth should be 20GB/s");
|
||||
TEST_ASSERT(constants.hwLatencies[NCCL_HW_NET][NCCL_ALGO_NVLS][NCCL_PROTO_SIMPLE] == 24.0, "NVLSTree base network latency should be 24us");
|
||||
|
||||
// Clean up
|
||||
pluginFinalize(context);
|
||||
TEST_PASS();
|
||||
}
|
||||
|
||||
// Test runner function pointer type
|
||||
typedef int (*TestFunction)(void);
|
||||
|
||||
@@ -783,6 +904,8 @@ TestCase test_cases[] = {
|
||||
{"large-config", test_large_config, "Large configuration files (dynamic allocation)"},
|
||||
{"stress-config", test_very_large_config_stress, "Very large configuration stress test"},
|
||||
{"empty-config", test_empty_config, "Empty configuration file handling"},
|
||||
{"nvl-domain", test_nvl_domain_info, "NVL domain info handling"},
|
||||
{"constants", test_tuner_constants, "Tuner constants initialization"},
|
||||
{NULL, NULL, NULL} // End marker
|
||||
};
|
||||
|
||||
@@ -826,6 +949,7 @@ int main(int argc, char* argv[]) {
|
||||
if (argc == 1) {
|
||||
// No arguments - run all tests
|
||||
for (int i = 0; test_cases[i].name != NULL; i++) {
|
||||
printf("Running test: %s\n", test_cases[i].name);
|
||||
total++;
|
||||
passed += test_cases[i].func();
|
||||
}
|
||||
|
||||
Ссылка в новой задаче
Block a user