Added Functional Tests for CSV Tuner Plugin (#1968)
* Add functional tests for CSV Tuner Plugin * Updated directory structure * Updated and renamed directories * Updated csv conf files * Updated readme * Updated readme * Updated readme
This commit is contained in:
Kapil S. Pawar
GitHub
@@ -0,0 +1,109 @@
|
||||
# *************************************************************************
|
||||
# * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
|
||||
# *
|
||||
# * See LICENSE.txt for license information
|
||||
# ************************************************************************
|
||||
|
||||
import os
|
||||
import pytest
|
||||
import subprocess
|
||||
import re
|
||||
from types import SimpleNamespace
|
||||
|
||||
WORKDIR = os.getcwd()
|
||||
|
||||
RCCL_INSTALL_DIR = "path/to/rccl"
|
||||
OMPI_INSTALL_DIR = "path/to/ompi/install"
|
||||
RCCL_TESTS_DIR = "path/to/rccl-tests"
|
||||
|
||||
# Plugin Paths
|
||||
PLUGIN_DIR = f"{RCCL_INSTALL_DIR}/ext-tuner/example"
|
||||
PLUGIN_SO = f"{PLUGIN_DIR}/libnccl-tuner-example.so"
|
||||
|
||||
# CSV Configs
|
||||
VALID_CONFIG_WITH_WILDCARDS = os.path.join(WORKDIR, "assets/csv_confs/valid_config_with_wildcards.conf")
|
||||
VALID_CONFIG_WITHOUT_WILDCARDS = os.path.join(WORKDIR, "assets/csv_confs/valid_config_without_wildcards.conf")
|
||||
NO_MATCHING_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/no_matching_config.conf")
|
||||
INCORRECT_VALUES_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/incorrect_values_config.conf")
|
||||
UNSUPPORTED_ALGO_PROTO_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/unsupported_algo_proto_config.conf")
|
||||
SINGLENODE_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/singlenode_config.conf")
|
||||
MULTINODE_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/multinode_config.conf")
|
||||
|
||||
LOGDIR = os.path.join(WORKDIR, "logs")
|
||||
os.makedirs(LOGDIR, exist_ok=True)
|
||||
|
||||
# Helper Functions
|
||||
def get_avg_bus_bandwidth(log_content: str):
|
||||
"""Extract average bus bandwidth from RCCL test log"""
|
||||
pattern = r'#\s*Avg bus bandwidth\s*:\s*([\d.]+)'
|
||||
match = re.search(pattern, log_content, re.IGNORECASE)
|
||||
return float(match.group(1)) if match else None
|
||||
|
||||
def check_node_interface(node: str, interface: str) -> bool:
|
||||
"""Check if a node has the specified interface with an IP address"""
|
||||
try:
|
||||
cmd = ["ssh", "-o", "ConnectTimeout=5", "-o", "StrictHostKeyChecking=no",
|
||||
node, f"ip addr show {interface} | grep 'inet ' | wc -l"]
|
||||
result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
||||
return result.returncode == 0 and int(result.stdout.strip()) > 0
|
||||
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, ValueError):
|
||||
return False
|
||||
|
||||
def find_common_interface(nodelist):
|
||||
"""Find a common network interface across all nodes"""
|
||||
interfaces_to_check = ["eth0", "eth1"]
|
||||
|
||||
for interface in interfaces_to_check:
|
||||
all_nodes_have_interface = True
|
||||
for node in nodelist:
|
||||
if not check_node_interface(node, interface):
|
||||
all_nodes_have_interface = False
|
||||
break
|
||||
if all_nodes_have_interface:
|
||||
return interface
|
||||
return None
|
||||
|
||||
def get_available_nodes():
|
||||
"""Get available nodes from SLURM environment"""
|
||||
try:
|
||||
# Get available nodes
|
||||
result = subprocess.run(
|
||||
["scontrol", "show", "hostnames"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
nodelist = result.stdout.strip().split('\n')
|
||||
nodelist = [node.strip() for node in nodelist if node.strip()]
|
||||
|
||||
return nodelist
|
||||
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
return []
|
||||
|
||||
# Pytest Fixture
|
||||
@pytest.fixture(scope="session")
|
||||
def paths():
|
||||
return SimpleNamespace(
|
||||
# Paths
|
||||
WORKDIR=WORKDIR,
|
||||
RCCL_INSTALL_DIR=RCCL_INSTALL_DIR,
|
||||
OMPI_INSTALL_DIR=OMPI_INSTALL_DIR,
|
||||
PLUGIN_DIR=PLUGIN_DIR,
|
||||
PLUGIN_SO=PLUGIN_SO,
|
||||
RCCL_TESTS_DIR=RCCL_TESTS_DIR,
|
||||
# CSV Configs
|
||||
VALID_CONFIG_WITH_WILDCARDS=VALID_CONFIG_WITH_WILDCARDS,
|
||||
VALID_CONFIG_WITHOUT_WILDCARDS=VALID_CONFIG_WITHOUT_WILDCARDS,
|
||||
NO_MATCHING_CONFIG=NO_MATCHING_CONFIG,
|
||||
INCORRECT_VALUES_CONFIG=INCORRECT_VALUES_CONFIG,
|
||||
UNSUPPORTED_ALGO_PROTO_CONFIG=UNSUPPORTED_ALGO_PROTO_CONFIG,
|
||||
SINGLENODE_CONFIG=SINGLENODE_CONFIG,
|
||||
MULTINODE_CONFIG=MULTINODE_CONFIG,
|
||||
LOGDIR=LOGDIR,
|
||||
# Helper Functions
|
||||
get_avg_bus_bandwidth=get_avg_bus_bandwidth,
|
||||
check_node_interface=check_node_interface,
|
||||
find_common_interface=find_common_interface,
|
||||
get_available_nodes=get_available_nodes,
|
||||
)
|
||||
@@ -0,0 +1,440 @@
|
||||
# *************************************************************************
|
||||
# * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
|
||||
# *
|
||||
# * See LICENSE.txt for license information
|
||||
# ************************************************************************
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import pytest
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allgather
|
||||
def test_valid_config_with_wildcards(paths):
|
||||
"""Test CSV plugin with wildcard values for matching configurations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.VALID_CONFIG_WITH_WILDCARDS,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_gather_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allgather_log_dir = os.path.join(paths.LOGDIR, "allgather_csv_plugin_test_logs")
|
||||
os.makedirs(allgather_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allgather_log_dir, "test_allgather_valid_config_with_wildcards.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin allgather test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.VALID_CONFIG_WITH_WILDCARDS}"
|
||||
|
||||
# Check that plugin applied valid configurations (test fails if no configs applied)
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied valid configurations, but none were applied. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allgather
|
||||
def test_valid_config_without_wildcards(paths):
|
||||
"""Test CSV plugin with specific values (no wildcards -1) for precise matching"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.VALID_CONFIG_WITHOUT_WILDCARDS,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_gather_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allgather_log_dir = os.path.join(paths.LOGDIR, "allgather_csv_plugin_test_logs")
|
||||
os.makedirs(allgather_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allgather_log_dir, "test_allgather_valid_config_without_wildcards.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin allgather test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.VALID_CONFIG_WITHOUT_WILDCARDS}"
|
||||
|
||||
# With specific values, plugin should either apply matching configs or report no matches
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
# Test should fail if no config is applied - we expect specific configs to match
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied at least one configuration from {paths.VALID_CONFIG_WITHOUT_WILDCARDS}. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allgather
|
||||
def test_no_matching_config(paths):
|
||||
"""Test CSV plugin behavior with no matching configurations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.NO_MATCHING_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "8",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_gather_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allgather_log_dir = os.path.join(paths.LOGDIR, "allgather_csv_plugin_test_logs")
|
||||
os.makedirs(allgather_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allgather_log_dir, "test_allgather_no_matching_config.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin allgather test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.NO_MATCHING_CONFIG}"
|
||||
|
||||
# Check that NO configurations were applied (they should not match the test environment)
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert not plugin_applied, \
|
||||
f"Plugin should NOT have applied any configurations from {paths.NO_MATCHING_CONFIG} as they don't match the test environment. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allgather
|
||||
def test_incorrect_values_config(paths):
|
||||
"""Test CSV plugin behavior with invalid/incorrect values in configuration"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.INCORRECT_VALUES_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_gather_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allgather_log_dir = os.path.join(paths.LOGDIR, "allgather_csv_plugin_test_logs")
|
||||
os.makedirs(allgather_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allgather_log_dir, "test_allgather_incorrect_values_config.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin allgather test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded some configurations (plugin should handle invalid values gracefully)
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.INCORRECT_VALUES_CONFIG}"
|
||||
|
||||
# Plugin should still function despite invalid values (using defaults)
|
||||
# It might apply configs with default values or report no matches
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
"Plugin should either apply configurations (with defaults) or report no matches"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allgather
|
||||
def test_unsupported_algo_proto_config(paths):
|
||||
"""Test that plugin handles unsupported algorithm/protocol combinations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.UNSUPPORTED_ALGO_PROTO_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_gather_perf",
|
||||
"-b", "64",
|
||||
"-e", "1M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allgather_log_dir = os.path.join(paths.LOGDIR, "allgather_csv_plugin_test_logs")
|
||||
os.makedirs(allgather_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allgather_log_dir, "test_allgather_unsupported_algo_proto.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin allgather test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.UNSUPPORTED_ALGO_PROTO_CONFIG}"
|
||||
|
||||
# Check for unsupported combinations - should see IGNORE or out of bounds messages
|
||||
ignored_combinations = "Algorithm/protocol combination" in log_content and "is marked as IGNORE" in log_content
|
||||
out_of_bounds = "out of bounds" in log_content
|
||||
|
||||
assert ignored_combinations or out_of_bounds, \
|
||||
f"Plugin should report unsupported algorithm/protocol combinations as IGNORE or out of bounds. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allgather
|
||||
def test_singlenode_config(paths):
|
||||
"""Test CSV plugin with single-node configuration"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.SINGLENODE_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "8",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_gather_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allgather_log_dir = os.path.join(paths.LOGDIR, "allgather_csv_plugin_test_logs")
|
||||
os.makedirs(allgather_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allgather_log_dir, "test_allgather_singlenode.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"Single-node CSV Plugin allgather test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.SINGLENODE_CONFIG}"
|
||||
|
||||
# Check that configurations were applied for single-node setup
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied single-node configurations from {paths.SINGLENODE_CONFIG}. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allgather
|
||||
@pytest.mark.multinode
|
||||
def test_multinode_config(paths):
|
||||
"""Test CSV plugin with multi-node configuration"""
|
||||
|
||||
try:
|
||||
# Get available nodes
|
||||
result = subprocess.run(
|
||||
["scontrol", "show", "hostnames"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True
|
||||
)
|
||||
nodelist = result.stdout.strip().split('\n')
|
||||
nodelist = [node.strip() for node in nodelist if node.strip()]
|
||||
|
||||
print(f"Available nodes: {nodelist}")
|
||||
|
||||
# Skip test if less than 2 nodes available
|
||||
if len(nodelist) < 2:
|
||||
pytest.skip(f"Multinode allgather test requires at least 2 nodes, but only {len(nodelist)} available: {nodelist}")
|
||||
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
pytest.skip("Multinode allgather test requires SLURM environment (scontrol command not available)")
|
||||
|
||||
# Check for common network interface across all nodes
|
||||
common_interface = paths.find_common_interface(nodelist)
|
||||
if common_interface is None:
|
||||
pytest.skip(f"Multinode allgather test requires all nodes to have the same network interface (eth0 or eth1).")
|
||||
|
||||
# Build host specification string (4 processes per node)
|
||||
host_spec = ",".join([f"{node}:8" for node in nodelist])
|
||||
total_processes = len(nodelist) * 8
|
||||
print(f"Using host specification: {host_spec}")
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_IGNORE_CPU_AFFINITY": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.MULTINODE_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
"NCCL_SOCKET_IFNAME": common_interface,
|
||||
"NCCL_DMABUF_ENABLE": "1",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", f"{total_processes}",
|
||||
"--host", host_spec,
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_gather_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allgather_log_dir = os.path.join(paths.LOGDIR, "allgather_csv_plugin_test_logs")
|
||||
os.makedirs(allgather_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allgather_log_dir, "test_allgather_multinode.log")
|
||||
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"Multi-node CSV Plugin allgather test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.MULTINODE_CONFIG}"
|
||||
|
||||
# Check that configurations were applied for multi-node setup
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied multi-node configurations from {paths.MULTINODE_CONFIG}. Check {log_file} for details"
|
||||
@@ -0,0 +1,431 @@
|
||||
# *************************************************************************
|
||||
# * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
|
||||
# *
|
||||
# * See LICENSE.txt for license information
|
||||
# ************************************************************************
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import pytest
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allreduce
|
||||
def test_valid_config_with_wildcards(paths):
|
||||
"""Test CSV plugin with wildcard values for matching configurations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.VALID_CONFIG_WITH_WILDCARDS,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allreduce_log_dir = os.path.join(paths.LOGDIR, "allreduce_csv_plugin_test_logs")
|
||||
os.makedirs(allreduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allreduce_log_dir, "test_allreduce_valid_config_with_wildcards.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.VALID_CONFIG_WITH_WILDCARDS}"
|
||||
|
||||
# Check that plugin applied valid configurations (test fails if no configs applied)
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied valid configurations, but none were applied. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allreduce
|
||||
def test_valid_config_without_wildcards(paths):
|
||||
"""Test CSV plugin with specific values (no wildcards -1) for precise matching"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.VALID_CONFIG_WITHOUT_WILDCARDS,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allreduce_log_dir = os.path.join(paths.LOGDIR, "allreduce_csv_plugin_test_logs")
|
||||
os.makedirs(allreduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allreduce_log_dir, "test_allreduce_valid_config_without_wildcards.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.VALID_CONFIG_WITHOUT_WILDCARDS}"
|
||||
|
||||
# With specific values, plugin should either apply matching configs or report no matches
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
# Test should fail if no config is applied - we expect specific configs to match
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied at least one configuration from {paths.VALID_CONFIG_WITHOUT_WILDCARDS}. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allreduce
|
||||
def test_no_matching_config(paths):
|
||||
"""Test CSV plugin behavior with no matching configurations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.NO_MATCHING_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "8",
|
||||
"--bind-to", "none",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allreduce_log_dir = os.path.join(paths.LOGDIR, "allreduce_csv_plugin_test_logs")
|
||||
os.makedirs(allreduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allreduce_log_dir, "test_allreduce_no_matching_config.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.NO_MATCHING_CONFIG}"
|
||||
|
||||
# Check that NO configurations were applied (they should not match the test environment)
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert not plugin_applied, \
|
||||
f"Plugin should NOT have applied any configurations from {paths.NO_MATCHING_CONFIG} as they don't match the test environment. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allreduce
|
||||
def test_incorrect_values_config(paths):
|
||||
"""Test CSV plugin behavior with invalid/incorrect values in configuration"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.INCORRECT_VALUES_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allreduce_log_dir = os.path.join(paths.LOGDIR, "allreduce_csv_plugin_test_logs")
|
||||
os.makedirs(allreduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allreduce_log_dir, "test_allreduce_incorrect_values_config.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded some configurations (plugin should handle invalid values gracefully)
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.INCORRECT_VALUES_CONFIG}"
|
||||
|
||||
# Plugin should still function despite invalid values (using defaults)
|
||||
# It might apply configs with default values or report no matches
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
"Plugin should either apply configurations (with defaults) or report no matches"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allreduce
|
||||
def test_unsupported_algo_proto_config(paths):
|
||||
"""Test that plugin handles unsupported algorithm/protocol combinations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.UNSUPPORTED_ALGO_PROTO_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allreduce_log_dir = os.path.join(paths.LOGDIR, "allreduce_csv_plugin_test_logs")
|
||||
os.makedirs(allreduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allreduce_log_dir, "test_allreduce_unsupported_algo_proto.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.UNSUPPORTED_ALGO_PROTO_CONFIG}"
|
||||
|
||||
# Check for unsupported combinations - should see IGNORE or out of bounds messages
|
||||
ignored_combinations = "Algorithm/protocol combination" in log_content and "is marked as IGNORE" in log_content
|
||||
out_of_bounds = "out of bounds" in log_content
|
||||
|
||||
assert ignored_combinations or out_of_bounds, \
|
||||
f"Plugin should report unsupported algorithm/protocol combinations as IGNORE or out of bounds. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allreduce
|
||||
def test_singlenode_config(paths):
|
||||
"""Test CSV plugin with single-node configuration"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.SINGLENODE_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "8",
|
||||
"--bind-to", "none",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allreduce_log_dir = os.path.join(paths.LOGDIR, "allreduce_csv_plugin_test_logs")
|
||||
os.makedirs(allreduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allreduce_log_dir, "test_allreduce_singlenode.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"Single-node CSV Plugin test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.SINGLENODE_CONFIG}"
|
||||
|
||||
# Check that configurations were applied for single-node setup
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied single-node configurations from {paths.SINGLENODE_CONFIG}. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.allreduce
|
||||
@pytest.mark.multinode
|
||||
def test_multinode_config(paths):
|
||||
"""Test CSV plugin with multi-node configuration"""
|
||||
|
||||
# Get available nodes using the shared function
|
||||
nodelist = paths.get_available_nodes()
|
||||
|
||||
# Skip test if no nodes available (SLURM not available) or less than 2 nodes
|
||||
if len(nodelist) == 0:
|
||||
pytest.skip("No nodes available")
|
||||
elif len(nodelist) < 2:
|
||||
pytest.skip(f"Multinode allreduce test requires at least 2 nodes, but only {len(nodelist)} available: {nodelist}")
|
||||
|
||||
# Check for common network interface across all nodes
|
||||
common_interface = paths.find_common_interface(nodelist)
|
||||
if common_interface is None:
|
||||
pytest.skip(f"Multinode allreduce test requires all nodes to have the same network interface (eth0 or eth1).")
|
||||
|
||||
# Build host specification string (4 processes per node)
|
||||
host_spec = ",".join([f"{node}:8" for node in nodelist])
|
||||
total_processes = len(nodelist) * 8
|
||||
print(f"Using host specification: {host_spec}")
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_IGNORE_CPU_AFFINITY": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.MULTINODE_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
"NCCL_SOCKET_IFNAME": common_interface,
|
||||
"NCCL_DMABUF_ENABLE": "1",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", f"{total_processes}",
|
||||
"--host", host_spec,
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/all_reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
allreduce_log_dir = os.path.join(paths.LOGDIR, "allreduce_csv_plugin_test_logs")
|
||||
os.makedirs(allreduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(allreduce_log_dir, "test_allreduce_multinode.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"Multi-node CSV Plugin test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.MULTINODE_CONFIG}"
|
||||
|
||||
# Check that configurations were applied for multi-node setup
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied multi-node configurations from {paths.MULTINODE_CONFIG}. Check {log_file} for details"
|
||||
|
||||
@@ -0,0 +1,428 @@
|
||||
# *************************************************************************
|
||||
# * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
|
||||
# *
|
||||
# * See LICENSE.txt for license information
|
||||
# ************************************************************************
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import pytest
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.broadcast
|
||||
def test_valid_config_with_wildcards(paths):
|
||||
"""Test CSV plugin with wildcard values for matching configurations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.VALID_CONFIG_WITH_WILDCARDS,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/broadcast_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
broadcast_log_dir = os.path.join(paths.LOGDIR, "broadcast_csv_plugin_test_logs")
|
||||
os.makedirs(broadcast_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(broadcast_log_dir, "test_broadcast_valid_config_with_wildcards.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin broadcast test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.VALID_CONFIG_WITH_WILDCARDS}"
|
||||
|
||||
# Check that plugin applied valid configurations (test fails if no configs applied)
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied valid configurations, but none were applied. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.broadcast
|
||||
def test_valid_config_without_wildcards(paths):
|
||||
"""Test CSV plugin with specific values (no wildcards -1) for precise matching"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.VALID_CONFIG_WITHOUT_WILDCARDS,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/broadcast_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
broadcast_log_dir = os.path.join(paths.LOGDIR, "broadcast_csv_plugin_test_logs")
|
||||
os.makedirs(broadcast_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(broadcast_log_dir, "test_broadcast_valid_config_without_wildcards.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin broadcast test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.VALID_CONFIG_WITHOUT_WILDCARDS}"
|
||||
|
||||
# With specific values, plugin should either apply matching configs or report no matches
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
# Test should fail if no config is applied - we expect specific configs to match
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied at least one configuration from {paths.VALID_CONFIG_WITHOUT_WILDCARDS}. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.broadcast
|
||||
def test_no_matching_config(paths):
|
||||
"""Test CSV plugin behavior with no matching configurations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.NO_MATCHING_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "8",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/broadcast_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
broadcast_log_dir = os.path.join(paths.LOGDIR, "broadcast_csv_plugin_test_logs")
|
||||
os.makedirs(broadcast_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(broadcast_log_dir, "test_broadcast_no_matching_config.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin broadcast test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.NO_MATCHING_CONFIG}"
|
||||
|
||||
# Check that NO configurations were applied (they should not match the test environment)
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert not plugin_applied, \
|
||||
f"Plugin should NOT have applied any configurations from {paths.NO_MATCHING_CONFIG} as they don't match the test environment. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.broadcast
|
||||
def test_incorrect_values_config(paths):
|
||||
"""Test CSV plugin behavior with invalid/incorrect values in configuration"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.INCORRECT_VALUES_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/broadcast_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
broadcast_log_dir = os.path.join(paths.LOGDIR, "broadcast_csv_plugin_test_logs")
|
||||
os.makedirs(broadcast_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(broadcast_log_dir, "test_broadcast_incorrect_values_config.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin broadcast test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded some configurations (plugin should handle invalid values gracefully)
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.INCORRECT_VALUES_CONFIG}"
|
||||
|
||||
# Plugin should still function despite invalid values (using defaults)
|
||||
# It might apply configs with default values or report no matches
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
"Plugin should either apply configurations (with defaults) or report no matches"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.broadcast
|
||||
def test_unsupported_algo_proto_config(paths):
|
||||
"""Test that plugin handles unsupported algorithm/protocol combinations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.UNSUPPORTED_ALGO_PROTO_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/broadcast_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
broadcast_log_dir = os.path.join(paths.LOGDIR, "broadcast_csv_plugin_test_logs")
|
||||
os.makedirs(broadcast_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(broadcast_log_dir, "test_broadcast_unsupported_algo_proto.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin broadcast test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.UNSUPPORTED_ALGO_PROTO_CONFIG}"
|
||||
|
||||
# Check for unsupported combinations - should see IGNORE or out of bounds messages
|
||||
ignored_combinations = "Algorithm/protocol combination" in log_content and "is marked as IGNORE" in log_content
|
||||
out_of_bounds = "out of bounds" in log_content
|
||||
|
||||
assert ignored_combinations or out_of_bounds, \
|
||||
f"Plugin should report unsupported algorithm/protocol combinations as IGNORE or out of bounds. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.broadcast
|
||||
def test_singlenode_config(paths):
|
||||
"""Test CSV plugin with single-node configuration"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.SINGLENODE_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "8",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/broadcast_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
broadcast_log_dir = os.path.join(paths.LOGDIR, "broadcast_csv_plugin_test_logs")
|
||||
os.makedirs(broadcast_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(broadcast_log_dir, "test_broadcast_singlenode.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"Single-node CSV Plugin broadcast test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.SINGLENODE_CONFIG}"
|
||||
|
||||
# Check that configurations were applied for single-node setup
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied single-node configurations from {paths.SINGLENODE_CONFIG}. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.broadcast
|
||||
@pytest.mark.multinode
|
||||
def test_multinode_config(paths):
|
||||
"""Test CSV plugin with multi-node configuration"""
|
||||
|
||||
# Get available nodes using the shared function
|
||||
nodelist = paths.get_available_nodes()
|
||||
|
||||
# Skip test if no nodes available (SLURM not available) or less than 2 nodes
|
||||
if len(nodelist) == 0:
|
||||
pytest.skip("No nodes available")
|
||||
elif len(nodelist) < 2:
|
||||
pytest.skip(f"Multinode broadcast test requires at least 2 nodes, but only {len(nodelist)} available: {nodelist}")
|
||||
|
||||
# Check for common network interface across all nodes
|
||||
common_interface = paths.find_common_interface(nodelist)
|
||||
if common_interface is None:
|
||||
pytest.skip(f"Multinode broadcast test requires all nodes to have the same network interface (eth0 or eth1).")
|
||||
|
||||
# Build host specification string (4 processes per node)
|
||||
host_spec = ",".join([f"{node}:8" for node in nodelist])
|
||||
total_processes = len(nodelist) * 8
|
||||
print(f"Using host specification: {host_spec}")
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_IGNORE_CPU_AFFINITY": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.MULTINODE_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
"NCCL_SOCKET_IFNAME": common_interface,
|
||||
"NCCL_DMABUF_ENABLE": "1",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", f"{total_processes}",
|
||||
"--host", host_spec,
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/broadcast_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
broadcast_log_dir = os.path.join(paths.LOGDIR, "broadcast_csv_plugin_test_logs")
|
||||
os.makedirs(broadcast_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(broadcast_log_dir, "test_broadcast_multinode.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"Multi-node CSV Plugin broadcast test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.MULTINODE_CONFIG}"
|
||||
|
||||
# Check that configurations were applied for multi-node setup
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied multi-node configurations from {paths.MULTINODE_CONFIG}. Check {log_file} for details"
|
||||
@@ -0,0 +1,428 @@
|
||||
# *************************************************************************
|
||||
# * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
|
||||
# *
|
||||
# * See LICENSE.txt for license information
|
||||
# ************************************************************************
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import pytest
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reduce
|
||||
def test_valid_config_with_wildcards(paths):
|
||||
"""Test CSV plugin with wildcard values for matching configurations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.VALID_CONFIG_WITH_WILDCARDS,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reduce_log_dir = os.path.join(paths.LOGDIR, "reduce_csv_plugin_test_logs")
|
||||
os.makedirs(reduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reduce_log_dir, "test_reduce_valid_config_with_wildcards.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin reduce test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.VALID_CONFIG_WITH_WILDCARDS}"
|
||||
|
||||
# Check that plugin applied valid configurations (test fails if no configs applied)
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied valid configurations, but none were applied. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reduce
|
||||
def test_valid_config_without_wildcards(paths):
|
||||
"""Test CSV plugin with specific values (no wildcards -1) for precise matching"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.VALID_CONFIG_WITHOUT_WILDCARDS,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reduce_log_dir = os.path.join(paths.LOGDIR, "reduce_csv_plugin_test_logs")
|
||||
os.makedirs(reduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reduce_log_dir, "test_reduce_valid_config_without_wildcards.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin reduce test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.VALID_CONFIG_WITHOUT_WILDCARDS}"
|
||||
|
||||
# With specific values, plugin should either apply matching configs or report no matches
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
# Test should fail if no config is applied - we expect specific configs to match
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied at least one configuration from {paths.VALID_CONFIG_WITHOUT_WILDCARDS}. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reduce
|
||||
def test_no_matching_config(paths):
|
||||
"""Test CSV plugin behavior with no matching configurations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.NO_MATCHING_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "8",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reduce_log_dir = os.path.join(paths.LOGDIR, "reduce_csv_plugin_test_logs")
|
||||
os.makedirs(reduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reduce_log_dir, "test_reduce_no_matching_config.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin reduce test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.NO_MATCHING_CONFIG}"
|
||||
|
||||
# Check that NO configurations were applied (they should not match the test environment)
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert not plugin_applied, \
|
||||
f"Plugin should NOT have applied any configurations from {paths.NO_MATCHING_CONFIG} as they don't match the test environment. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reduce
|
||||
def test_incorrect_values_config(paths):
|
||||
"""Test CSV plugin behavior with invalid/incorrect values in configuration"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.INCORRECT_VALUES_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reduce_log_dir = os.path.join(paths.LOGDIR, "reduce_csv_plugin_test_logs")
|
||||
os.makedirs(reduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reduce_log_dir, "test_reduce_incorrect_values_config.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin reduce test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded some configurations (plugin should handle invalid values gracefully)
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.INCORRECT_VALUES_CONFIG}"
|
||||
|
||||
# Plugin should still function despite invalid values (using defaults)
|
||||
# It might apply configs with default values or report no matches
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
"Plugin should either apply configurations (with defaults) or report no matches"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reduce
|
||||
def test_unsupported_algo_proto_config(paths):
|
||||
"""Test that plugin handles unsupported algorithm/protocol combinations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.UNSUPPORTED_ALGO_PROTO_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reduce_log_dir = os.path.join(paths.LOGDIR, "reduce_csv_plugin_test_logs")
|
||||
os.makedirs(reduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reduce_log_dir, "test_reduce_unsupported_algo_proto.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin reduce test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.UNSUPPORTED_ALGO_PROTO_CONFIG}"
|
||||
|
||||
# Check for unsupported combinations - should see IGNORE or out of bounds messages
|
||||
ignored_combinations = "Algorithm/protocol combination" in log_content and "is marked as IGNORE" in log_content
|
||||
out_of_bounds = "out of bounds" in log_content
|
||||
|
||||
assert ignored_combinations or out_of_bounds, \
|
||||
f"Plugin should report unsupported algorithm/protocol combinations as IGNORE or out of bounds. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reduce
|
||||
def test_singlenode_config(paths):
|
||||
"""Test CSV plugin with single-node configuration"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.SINGLENODE_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "8",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reduce_log_dir = os.path.join(paths.LOGDIR, "reduce_csv_plugin_test_logs")
|
||||
os.makedirs(reduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reduce_log_dir, "test_reduce_singlenode.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"Single-node CSV Plugin reduce test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.SINGLENODE_CONFIG}"
|
||||
|
||||
# Check that configurations were applied for single-node setup
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied single-node configurations from {paths.SINGLENODE_CONFIG}. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reduce
|
||||
@pytest.mark.multinode
|
||||
def test_multinode_config(paths):
|
||||
"""Test CSV plugin with multi-node configuration"""
|
||||
|
||||
# Get available nodes using the shared function
|
||||
nodelist = paths.get_available_nodes()
|
||||
|
||||
# Skip test if no nodes available (SLURM not available) or less than 2 nodes
|
||||
if len(nodelist) == 0:
|
||||
pytest.skip("No nodes available")
|
||||
elif len(nodelist) < 2:
|
||||
pytest.skip(f"Multinode reduce test requires at least 2 nodes, but only {len(nodelist)} available: {nodelist}")
|
||||
|
||||
# Check for common network interface across all nodes
|
||||
common_interface = paths.find_common_interface(nodelist)
|
||||
if common_interface is None:
|
||||
pytest.skip(f"Multinode reduce test requires all nodes to have the same network interface (eth0 or eth1).")
|
||||
|
||||
# Build host specification string (4 processes per node)
|
||||
host_spec = ",".join([f"{node}:8" for node in nodelist])
|
||||
total_processes = len(nodelist) * 8
|
||||
print(f"Using host specification: {host_spec}")
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_IGNORE_CPU_AFFINITY": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.MULTINODE_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
"NCCL_SOCKET_IFNAME": common_interface,
|
||||
"NCCL_DMABUF_ENABLE": "1",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", f"{total_processes}",
|
||||
"--host", host_spec,
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_perf",
|
||||
"-b", "8", # 64 bytes start (faster than 1 byte)
|
||||
"-e", "128M", # 64MB end (much smaller than 16GB)
|
||||
"-f", "2", # factor 4 (fewer size steps)
|
||||
"-g", "1", # gap 1
|
||||
]
|
||||
|
||||
reduce_log_dir = os.path.join(paths.LOGDIR, "reduce_csv_plugin_test_logs")
|
||||
os.makedirs(reduce_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reduce_log_dir, "test_reduce_multinode.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"Multi-node CSV Plugin reduce test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.MULTINODE_CONFIG}"
|
||||
|
||||
# Check that configurations were applied for multi-node setup
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied multi-node configurations from {paths.MULTINODE_CONFIG}. Check {log_file} for details"
|
||||
@@ -0,0 +1,428 @@
|
||||
# *************************************************************************
|
||||
# * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
|
||||
# *
|
||||
# * See LICENSE.txt for license information
|
||||
# ************************************************************************
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import pytest
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reducescatter
|
||||
def test_valid_config_with_wildcards(paths):
|
||||
"""Test CSV plugin with wildcard values for matching configurations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.VALID_CONFIG_WITH_WILDCARDS,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_scatter_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reducescatter_log_dir = os.path.join(paths.LOGDIR, "reducescatter_csv_plugin_test_logs")
|
||||
os.makedirs(reducescatter_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reducescatter_log_dir, "test_reducescatter_valid_config_with_wildcards.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin reducescatter test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.VALID_CONFIG_WITH_WILDCARDS}"
|
||||
|
||||
# Check that plugin applied valid configurations (test fails if no configs applied)
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied valid configurations, but none were applied. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reducescatter
|
||||
def test_valid_config_without_wildcards(paths):
|
||||
"""Test CSV plugin with specific values (no wildcards -1) for precise matching"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.VALID_CONFIG_WITHOUT_WILDCARDS,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_scatter_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reducescatter_log_dir = os.path.join(paths.LOGDIR, "reducescatter_csv_plugin_test_logs")
|
||||
os.makedirs(reducescatter_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reducescatter_log_dir, "test_reducescatter_valid_config_without_wildcards.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin reducescatter test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.VALID_CONFIG_WITHOUT_WILDCARDS}"
|
||||
|
||||
# With specific values, plugin should either apply matching configs or report no matches
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
# Test should fail if no config is applied - we expect specific configs to match
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied at least one configuration from {paths.VALID_CONFIG_WITHOUT_WILDCARDS}. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reducescatter
|
||||
def test_no_matching_config(paths):
|
||||
"""Test CSV plugin behavior with no matching configurations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.NO_MATCHING_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "8",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_scatter_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reducescatter_log_dir = os.path.join(paths.LOGDIR, "reducescatter_csv_plugin_test_logs")
|
||||
os.makedirs(reducescatter_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reducescatter_log_dir, "test_reducescatter_no_matching_config.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin reducescatter test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.NO_MATCHING_CONFIG}"
|
||||
|
||||
# Check that NO configurations were applied (they should not match the test environment)
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert not plugin_applied, \
|
||||
f"Plugin should NOT have applied any configurations from {paths.NO_MATCHING_CONFIG} as they don't match the test environment. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reducescatter
|
||||
def test_incorrect_values_config(paths):
|
||||
"""Test CSV plugin behavior with invalid/incorrect values in configuration"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.INCORRECT_VALUES_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_scatter_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reducescatter_log_dir = os.path.join(paths.LOGDIR, "reducescatter_csv_plugin_test_logs")
|
||||
os.makedirs(reducescatter_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reducescatter_log_dir, "test_reducescatter_incorrect_values_config.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin reducescatter test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded some configurations (plugin should handle invalid values gracefully)
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.INCORRECT_VALUES_CONFIG}"
|
||||
|
||||
# Plugin should still function despite invalid values (using defaults)
|
||||
# It might apply configs with default values or report no matches
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
"Plugin should either apply configurations (with defaults) or report no matches"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reducescatter
|
||||
def test_unsupported_algo_proto_config(paths):
|
||||
"""Test that plugin handles unsupported algorithm/protocol combinations"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.UNSUPPORTED_ALGO_PROTO_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "4",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_scatter_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reducescatter_log_dir = os.path.join(paths.LOGDIR, "reducescatter_csv_plugin_test_logs")
|
||||
os.makedirs(reducescatter_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reducescatter_log_dir, "test_reducescatter_unsupported_algo_proto.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"CSV Plugin reducescatter test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.UNSUPPORTED_ALGO_PROTO_CONFIG}"
|
||||
|
||||
# Check for unsupported combinations - should see IGNORE or out of bounds messages
|
||||
ignored_combinations = "Algorithm/protocol combination" in log_content and "is marked as IGNORE" in log_content
|
||||
out_of_bounds = "out of bounds" in log_content
|
||||
|
||||
assert ignored_combinations or out_of_bounds, \
|
||||
f"Plugin should report unsupported algorithm/protocol combinations as IGNORE or out of bounds. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reducescatter
|
||||
def test_singlenode_config(paths):
|
||||
"""Test CSV plugin with single-node configuration"""
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.SINGLENODE_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", "8",
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_scatter_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reducescatter_log_dir = os.path.join(paths.LOGDIR, "reducescatter_csv_plugin_test_logs")
|
||||
os.makedirs(reducescatter_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reducescatter_log_dir, "test_reducescatter_singlenode.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"Single-node CSV Plugin reducescatter test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.SINGLENODE_CONFIG}"
|
||||
|
||||
# Check that configurations were applied for single-node setup
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied single-node configurations from {paths.SINGLENODE_CONFIG}. Check {log_file} for details"
|
||||
|
||||
@pytest.mark.ext_tuner
|
||||
@pytest.mark.reducescatter
|
||||
@pytest.mark.multinode
|
||||
def test_multinode_config(paths):
|
||||
"""Test CSV plugin with multi-node configuration"""
|
||||
|
||||
# Get available nodes using the shared function
|
||||
nodelist = paths.get_available_nodes()
|
||||
|
||||
# Skip test if no nodes available (SLURM not available) or less than 2 nodes
|
||||
if len(nodelist) == 0:
|
||||
pytest.skip("No nodes available")
|
||||
elif len(nodelist) < 2:
|
||||
pytest.skip(f"Multinode reducescatter test requires at least 2 nodes, but only {len(nodelist)} available: {nodelist}")
|
||||
|
||||
# Check for common network interface across all nodes
|
||||
common_interface = paths.find_common_interface(nodelist)
|
||||
if common_interface is None:
|
||||
pytest.skip(f"Multinode reducescatter test requires all nodes to have the same network interface (eth0 or eth1).")
|
||||
|
||||
# Build host specification string (4 processes per node)
|
||||
host_spec = ",".join([f"{node}:8" for node in nodelist])
|
||||
total_processes = len(nodelist) * 8
|
||||
print(f"Using host specification: {host_spec}")
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update({
|
||||
"PATH": f"{paths.OMPI_INSTALL_DIR}/bin:{env.get('PATH', '')}",
|
||||
"LD_LIBRARY_PATH": f"{paths.RCCL_INSTALL_DIR}:{paths.OMPI_INSTALL_DIR}/lib:{paths.PLUGIN_DIR}:{env.get('LD_LIBRARY_PATH', '')}",
|
||||
"HSA_NO_SCRATCH_RECLAIM": "1",
|
||||
"NCCL_IGNORE_CPU_AFFINITY": "1",
|
||||
"NCCL_TUNER_PLUGIN": paths.PLUGIN_SO,
|
||||
"NCCL_TUNER_CONFIG_FILE": paths.MULTINODE_CONFIG,
|
||||
"NCCL_DEBUG": "INFO",
|
||||
"NCCL_DEBUG_SUBSYS": "TUNING",
|
||||
"NCCL_SOCKET_IFNAME": common_interface,
|
||||
"NCCL_DMABUF_ENABLE": "1",
|
||||
})
|
||||
|
||||
args = [
|
||||
f"{paths.OMPI_INSTALL_DIR}/bin/mpirun", "-np", f"{total_processes}",
|
||||
"--host", host_spec,
|
||||
"--mca", "pml", "ucx",
|
||||
"--mca", "btl", "^vader,openib",
|
||||
f"{paths.RCCL_TESTS_DIR}/build/reduce_scatter_perf",
|
||||
"-b", "8",
|
||||
"-e", "128M",
|
||||
"-f", "2",
|
||||
"-g", "1",
|
||||
]
|
||||
|
||||
reducescatter_log_dir = os.path.join(paths.LOGDIR, "reducescatter_csv_plugin_test_logs")
|
||||
os.makedirs(reducescatter_log_dir, exist_ok=True)
|
||||
|
||||
log_file = os.path.join(reducescatter_log_dir, "test_reducescatter_multinode.log")
|
||||
with open(log_file, "w") as logfile:
|
||||
rccl_test = subprocess.run(
|
||||
args,
|
||||
env=env,
|
||||
stdout=logfile,
|
||||
stderr=subprocess.STDOUT,
|
||||
universal_newlines=True
|
||||
)
|
||||
|
||||
assert rccl_test.returncode == 0, f"Multi-node CSV Plugin reducescatter test failed, see {log_file}"
|
||||
|
||||
# Read and validate log content
|
||||
with open(log_file, "r") as logfile:
|
||||
log_content = logfile.read()
|
||||
|
||||
# Check that plugin loaded configurations
|
||||
assert "TUNER/ExamplePlugin: Loaded" in log_content and "tuning configurations" in log_content, \
|
||||
f"Plugin should have loaded configurations from {paths.MULTINODE_CONFIG}"
|
||||
|
||||
# Check that configurations were applied for multi-node setup
|
||||
plugin_applied = "TUNER/ExamplePlugin: Applied config for collType=" in log_content
|
||||
|
||||
assert plugin_applied, \
|
||||
f"Plugin should have applied multi-node configurations from {paths.MULTINODE_CONFIG}. Check {log_file} for details"
|
||||
Reference in New Issue
Block a user