Kapil S. Pawar
c8da880dc7
* Add functional tests for CSV Tuner Plugin * Updated directory structure * Updated and renamed directories * Updated csv conf files * Updated readme * Updated readme * Updated readme
109 rindas
4.1 KiB
Python
109 rindas
4.1 KiB
Python
# *************************************************************************
|
|
# * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
|
|
# *
|
|
# * See LICENSE.txt for license information
|
|
# ************************************************************************
|
|
|
|
import os
|
|
import pytest
|
|
import subprocess
|
|
import re
|
|
from types import SimpleNamespace
|
|
|
|
WORKDIR = os.getcwd()
|
|
|
|
RCCL_INSTALL_DIR = "path/to/rccl"
|
|
OMPI_INSTALL_DIR = "path/to/ompi/install"
|
|
RCCL_TESTS_DIR = "path/to/rccl-tests"
|
|
|
|
# Plugin Paths
|
|
PLUGIN_DIR = f"{RCCL_INSTALL_DIR}/ext-tuner/example"
|
|
PLUGIN_SO = f"{PLUGIN_DIR}/libnccl-tuner-example.so"
|
|
|
|
# CSV Configs
|
|
VALID_CONFIG_WITH_WILDCARDS = os.path.join(WORKDIR, "assets/csv_confs/valid_config_with_wildcards.conf")
|
|
VALID_CONFIG_WITHOUT_WILDCARDS = os.path.join(WORKDIR, "assets/csv_confs/valid_config_without_wildcards.conf")
|
|
NO_MATCHING_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/no_matching_config.conf")
|
|
INCORRECT_VALUES_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/incorrect_values_config.conf")
|
|
UNSUPPORTED_ALGO_PROTO_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/unsupported_algo_proto_config.conf")
|
|
SINGLENODE_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/singlenode_config.conf")
|
|
MULTINODE_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/multinode_config.conf")
|
|
|
|
LOGDIR = os.path.join(WORKDIR, "logs")
|
|
os.makedirs(LOGDIR, exist_ok=True)
|
|
|
|
# Helper Functions
|
|
def get_avg_bus_bandwidth(log_content: str):
|
|
"""Extract average bus bandwidth from RCCL test log"""
|
|
pattern = r'#\s*Avg bus bandwidth\s*:\s*([\d.]+)'
|
|
match = re.search(pattern, log_content, re.IGNORECASE)
|
|
return float(match.group(1)) if match else None
|
|
|
|
def check_node_interface(node: str, interface: str) -> bool:
|
|
"""Check if a node has the specified interface with an IP address"""
|
|
try:
|
|
cmd = ["ssh", "-o", "ConnectTimeout=5", "-o", "StrictHostKeyChecking=no",
|
|
node, f"ip addr show {interface} | grep 'inet ' | wc -l"]
|
|
result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
|
|
return result.returncode == 0 and int(result.stdout.strip()) > 0
|
|
except (subprocess.CalledProcessError, subprocess.TimeoutExpired, ValueError):
|
|
return False
|
|
|
|
def find_common_interface(nodelist):
|
|
"""Find a common network interface across all nodes"""
|
|
interfaces_to_check = ["eth0", "eth1"]
|
|
|
|
for interface in interfaces_to_check:
|
|
all_nodes_have_interface = True
|
|
for node in nodelist:
|
|
if not check_node_interface(node, interface):
|
|
all_nodes_have_interface = False
|
|
break
|
|
if all_nodes_have_interface:
|
|
return interface
|
|
return None
|
|
|
|
def get_available_nodes():
|
|
"""Get available nodes from SLURM environment"""
|
|
try:
|
|
# Get available nodes
|
|
result = subprocess.run(
|
|
["scontrol", "show", "hostnames"],
|
|
capture_output=True,
|
|
text=True,
|
|
check=True
|
|
)
|
|
nodelist = result.stdout.strip().split('\n')
|
|
nodelist = [node.strip() for node in nodelist if node.strip()]
|
|
|
|
return nodelist
|
|
|
|
except (subprocess.CalledProcessError, FileNotFoundError):
|
|
return []
|
|
|
|
# Pytest Fixture
|
|
@pytest.fixture(scope="session")
|
|
def paths():
|
|
return SimpleNamespace(
|
|
# Paths
|
|
WORKDIR=WORKDIR,
|
|
RCCL_INSTALL_DIR=RCCL_INSTALL_DIR,
|
|
OMPI_INSTALL_DIR=OMPI_INSTALL_DIR,
|
|
PLUGIN_DIR=PLUGIN_DIR,
|
|
PLUGIN_SO=PLUGIN_SO,
|
|
RCCL_TESTS_DIR=RCCL_TESTS_DIR,
|
|
# CSV Configs
|
|
VALID_CONFIG_WITH_WILDCARDS=VALID_CONFIG_WITH_WILDCARDS,
|
|
VALID_CONFIG_WITHOUT_WILDCARDS=VALID_CONFIG_WITHOUT_WILDCARDS,
|
|
NO_MATCHING_CONFIG=NO_MATCHING_CONFIG,
|
|
INCORRECT_VALUES_CONFIG=INCORRECT_VALUES_CONFIG,
|
|
UNSUPPORTED_ALGO_PROTO_CONFIG=UNSUPPORTED_ALGO_PROTO_CONFIG,
|
|
SINGLENODE_CONFIG=SINGLENODE_CONFIG,
|
|
MULTINODE_CONFIG=MULTINODE_CONFIG,
|
|
LOGDIR=LOGDIR,
|
|
# Helper Functions
|
|
get_avg_bus_bandwidth=get_avg_bus_bandwidth,
|
|
check_node_interface=check_node_interface,
|
|
find_common_interface=find_common_interface,
|
|
get_available_nodes=get_available_nodes,
|
|
) |