Added Functional Tests for CSV Tuner Plugin (#1968)

* Add functional tests for CSV Tuner Plugin * Updated directory structure * Updated and renamed directories * Updated csv conf files * Updated readme * Updated readme * Updated readme [ROCm/rccl commit: c8da880dc7]
2025-11-11 10:11:19 -06:00
parent 0d3fba9a22
commit c4d7680749
17 changed files with 2824 additions and 0 deletions
@@ -0,0 +1,109 @@
+# *************************************************************************
+#  * Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved.
+#  *
+#  * See LICENSE.txt for license information
+#  ************************************************************************
+
+import os
+import pytest
+import subprocess
+import re
+from types import SimpleNamespace
+
+WORKDIR = os.getcwd()
+
+RCCL_INSTALL_DIR = "path/to/rccl"
+OMPI_INSTALL_DIR = "path/to/ompi/install"
+RCCL_TESTS_DIR = "path/to/rccl-tests"
+
+# Plugin Paths
+PLUGIN_DIR = f"{RCCL_INSTALL_DIR}/ext-tuner/example"
+PLUGIN_SO = f"{PLUGIN_DIR}/libnccl-tuner-example.so"
+
+# CSV Configs 
+VALID_CONFIG_WITH_WILDCARDS = os.path.join(WORKDIR, "assets/csv_confs/valid_config_with_wildcards.conf")
+VALID_CONFIG_WITHOUT_WILDCARDS = os.path.join(WORKDIR, "assets/csv_confs/valid_config_without_wildcards.conf")
+NO_MATCHING_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/no_matching_config.conf")
+INCORRECT_VALUES_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/incorrect_values_config.conf")
+UNSUPPORTED_ALGO_PROTO_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/unsupported_algo_proto_config.conf")
+SINGLENODE_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/singlenode_config.conf")
+MULTINODE_CONFIG = os.path.join(WORKDIR, "assets/csv_confs/multinode_config.conf")
+
+LOGDIR = os.path.join(WORKDIR, "logs")
+os.makedirs(LOGDIR, exist_ok=True)
+
+# Helper Functions
+def get_avg_bus_bandwidth(log_content: str):
+    """Extract average bus bandwidth from RCCL test log"""
+    pattern = r'#\s*Avg bus bandwidth\s*:\s*([\d.]+)'
+    match = re.search(pattern, log_content, re.IGNORECASE)
+    return float(match.group(1)) if match else None
+
+def check_node_interface(node: str, interface: str) -> bool:
+    """Check if a node has the specified interface with an IP address"""
+    try:
+        cmd = ["ssh", "-o", "ConnectTimeout=5", "-o", "StrictHostKeyChecking=no",
+               node, f"ip addr show {interface} | grep 'inet ' | wc -l"]
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
+        return result.returncode == 0 and int(result.stdout.strip()) > 0
+    except (subprocess.CalledProcessError, subprocess.TimeoutExpired, ValueError):
+        return False
+
+def find_common_interface(nodelist):
+    """Find a common network interface across all nodes"""
+    interfaces_to_check = ["eth0", "eth1"]
+
+    for interface in interfaces_to_check:
+        all_nodes_have_interface = True
+        for node in nodelist:
+            if not check_node_interface(node, interface):
+                all_nodes_have_interface = False
+                break
+        if all_nodes_have_interface:
+            return interface
+    return None
+
+def get_available_nodes():
+    """Get available nodes from SLURM environment"""
+    try:
+        # Get available nodes
+        result = subprocess.run(
+            ["scontrol", "show", "hostnames"], 
+            capture_output=True, 
+            text=True, 
+            check=True
+        )
+        nodelist = result.stdout.strip().split('\n')
+        nodelist = [node.strip() for node in nodelist if node.strip()]
+        
+        return nodelist
+        
+    except (subprocess.CalledProcessError, FileNotFoundError):
+        return []
+
+# Pytest Fixture
+@pytest.fixture(scope="session")
+def paths():
+    return SimpleNamespace(
+        # Paths
+        WORKDIR=WORKDIR,
+        RCCL_INSTALL_DIR=RCCL_INSTALL_DIR,
+        OMPI_INSTALL_DIR=OMPI_INSTALL_DIR,
+        PLUGIN_DIR=PLUGIN_DIR,
+        PLUGIN_SO=PLUGIN_SO,
+        RCCL_TESTS_DIR=RCCL_TESTS_DIR,
+        # CSV Configs
+        VALID_CONFIG_WITH_WILDCARDS=VALID_CONFIG_WITH_WILDCARDS,
+        VALID_CONFIG_WITHOUT_WILDCARDS=VALID_CONFIG_WITHOUT_WILDCARDS,
+        NO_MATCHING_CONFIG=NO_MATCHING_CONFIG,
+        INCORRECT_VALUES_CONFIG=INCORRECT_VALUES_CONFIG,
+        UNSUPPORTED_ALGO_PROTO_CONFIG=UNSUPPORTED_ALGO_PROTO_CONFIG,
+        SINGLENODE_CONFIG=SINGLENODE_CONFIG,
+        MULTINODE_CONFIG=MULTINODE_CONFIG,
+        LOGDIR=LOGDIR,
+        # Helper Functions
+        get_avg_bus_bandwidth=get_avg_bus_bandwidth,
+        check_node_interface=check_node_interface,
+        find_common_interface=find_common_interface,
+        get_available_nodes=get_available_nodes,
+    )