From e5cc96bf4826997caada75875b5e711a54723904 Mon Sep 17 00:00:00 2001 From: Donato Capitella Date: Sun, 1 Feb 2026 15:43:56 +0000 Subject: [PATCH] feat: Introduce vLLM cluster benchmarking and setup scripts, and expand the list of models for local benchmarks. --- Dockerfile | 4 +- benchmarks/run_vllm_bench.py | 18 +- benchmarks/vllm_cluster_bench.py | 239 +++++++++++++++ scripts/99-toolbox-banner.sh | 2 + scripts/start_vllm_cluster.py | 501 +++++++++++++++++++++++++++++++ 5 files changed, 750 insertions(+), 14 deletions(-) create mode 100755 benchmarks/vllm_cluster_bench.py create mode 100755 scripts/start_vllm_cluster.py diff --git a/Dockerfile b/Dockerfile index 1fcd18d..9799be9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -126,9 +126,11 @@ COPY scripts/01-rocm-env-for-triton.sh /etc/profile.d/01-rocm-env-for-triton.sh COPY scripts/99-toolbox-banner.sh /etc/profile.d/99-toolbox-banner.sh COPY scripts/zz-venv-last.sh /etc/profile.d/zz-venv-last.sh COPY scripts/start_vllm.py /usr/local/bin/start-vllm +COPY scripts/start_vllm_cluster.py /usr/local/bin/start-vllm-cluster COPY benchmarks/max_context_results.json /opt/max_context_results.json COPY benchmarks/run_vllm_bench.py /opt/run_vllm_bench.py -RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm && chmod 0644 /opt/max_context_results.json +COPY benchmarks/vllm_cluster_bench.py /opt/vllm_cluster_bench.py +RUN chmod 0644 /etc/profile.d/*.sh && chmod +x /usr/local/bin/start-vllm && chmod +x /usr/local/bin/start-vllm-cluster && chmod +x /opt/vllm_cluster_bench.py && chmod 0644 /opt/max_context_results.json RUN chmod 0644 /etc/profile.d/*.sh RUN printf 'ulimit -S -c 0\n' > /etc/profile.d/90-nocoredump.sh && chmod 0644 /etc/profile.d/90-nocoredump.sh diff --git a/benchmarks/run_vllm_bench.py b/benchmarks/run_vllm_bench.py index 37b664a..3ca45b0 100644 --- a/benchmarks/run_vllm_bench.py +++ b/benchmarks/run_vllm_bench.py @@ -91,11 +91,11 @@ MODEL_TABLE = { } MODELS_TO_RUN = [ - #"meta-llama/Meta-Llama-3.1-8B-Instruct", - #"google/gemma-3-12b-it", - #"Qwen/Qwen3-14B-AWQ", - #"openai/gpt-oss-20b", - #"openai/gpt-oss-120b", + "meta-llama/Meta-Llama-3.1-8B-Instruct", + "google/gemma-3-12b-it", + "Qwen/Qwen3-14B-AWQ", + "openai/gpt-oss-20b", + "openai/gpt-oss-120b", "cpatonn/Qwen3-Coder-30B-A3B-Instruct-GPTQ-4bit", "dazipe/Qwen3-Next-80B-A3B-Instruct-GPTQ-Int4A16", ] @@ -112,14 +112,6 @@ def get_gpu_count(): # Output format: "GPU[0] : Device Name: ..." res = subprocess.run(["rocm-smi", "--showid"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if res.returncode == 0: - # Filter specifically for the target GPU as requested - # target_gpu = "AMD Radeon AI PRO R9700" - # count = 0 - # for line in res.stdout.strip().split('\n'): - # if "Device Name" in line and target_gpu in line: - # count += 1 - - # return count if count > 0 else 1 return 1 # Force return 1 for Strix Halo APU else: log("rocm-smi failed, defaulting to 1 GPU (Hardcoded Fallback)") diff --git a/benchmarks/vllm_cluster_bench.py b/benchmarks/vllm_cluster_bench.py new file mode 100755 index 0000000..5e48394 --- /dev/null +++ b/benchmarks/vllm_cluster_bench.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +import subprocess, time, json, sys, os, requests, argparse +from pathlib import Path + +# ========================= +# ⚙️ GLOBAL SETTINGS +# ========================= + +# CLUSTER CONFIG: 2x Strix Halo (TP=2) +# User requested specifically to test with TP=2 on the cluster. +CLUSTER_TP = 2 +GPU_UTIL = "0.90" + +# THROUGHPUT CONFIG (Same as run_vllm_bench) +OFF_NUM_PROMPTS = 200 +OFF_FORCED_OUTPUT = "512" +DEFAULT_BATCH_TOKENS = "8192" + +RESULTS_DIR = Path("cluster_benchmark_results") +RESULTS_DIR.mkdir(exist_ok=True) + +# Reuse the model table from the main benchmark script +# We can just import it or copy it. Importing is cleaner but might rely on path. +# For standalone robustness, I will copy the minimal needed config or import if possible. +# Since this is a new file in root/benchmarks? No, likely scripts/ or same dir. +# Let's assume it's in the same dir as run_vllm_bench.py. + +try: + from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN +except ImportError: + # Fallback if run directly and path issues + sys.path.append(os.path.dirname(__file__)) + from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN + +# ========================= +# UTILS (Adapted for Cluster) +# ========================= + +def log(msg): print(f"\n[CLUSTER-BENCH] {msg}") + +def check_ray_status(): + """Checks if Ray cluster is active with at least 2 nodes.""" + try: + res = subprocess.run(["ray", "status"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if res.returncode != 0: + return False + + # Basic check for 2 nodes + active_nodes = 0 + in_active_section = False + for line in res.stdout.splitlines(): + if "Active:" in line: + in_active_section = True + continue + if "Pending:" in line or "Recent failures:" in line: + in_active_section = False + + if in_active_section and line.strip().startswith("1 node_"): + active_nodes += 1 + + return active_nodes >= 2 + except: + return False + +def get_net_iface(ip_prefix="192.168.100"): + """ + Auto-detects the interface that serves the cluster network. + Assumes standard 192.168.100.x setup from start_vllm_cluster.py + """ + try: + # ip -o addr show | grep 192.168.100 + cmd = f"ip -o addr show | grep {ip_prefix}" + res = subprocess.check_output(cmd, shell=True, text=True).strip() + # Output format: 2: eth0 inet 192.168.100.1/24 ... + parts = res.split() + if len(parts) >= 2: + return parts[1] # Interface name + except: + pass + return "eth0" # Fallback + +def get_local_ip(iface): + try: + cmd = f"ip -o -4 addr show {iface} | awk '{{print $4}}' | cut -d/ -f1" + return subprocess.check_output(cmd, shell=True, text=True).strip() + except: + return "127.0.0.1" + +def nuke_vllm_cache(): + cache = Path.home() / ".cache" / "vllm" + if cache.exists(): + try: + print(f"Clearing vLLM cache...", end="", flush=True) + subprocess.run(["rm", "-rf", str(cache)], check=True) + cache.mkdir(parents=True, exist_ok=True) + print(" Done.") + time.sleep(2) + except: pass + +def get_dataset(): + # Same as original + data_path = Path("ShareGPT_V3_unfiltered_cleaned_split.json") + if data_path.exists(): return str(data_path) + + log("Downloading ShareGPT dataset...") + url = "https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json" + try: + r = requests.get(url, stream=True, timeout=15) + r.raise_for_status() + with open(data_path, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): f.write(chunk) + return str(data_path) + except Exception as e: + log(f"WARNING: ShareGPT download failed ({e}). using RANDOM.") + return None + +def get_cluster_env(): + # Detect Interface and IP + rdma_iface = get_net_iface() + host_ip = get_local_ip(rdma_iface) + + env = os.environ.copy() + + # Critical Cluster Envs (Match start_vllm_cluster.py) + env["RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES"] = "1" + env["VLLM_HOST_IP"] = host_ip + env["NCCL_SOCKET_IFNAME"] = rdma_iface + env["GLOO_SOCKET_IFNAME"] = rdma_iface + # RCCL specific + env["NCCL_IB_GID_INDEX"] = "1" + env["NCCL_IB_DISABLE"] = "0" + env["NCCL_NET_GDR_LEVEL"] = "0" + + return env + +def get_model_args(model): + config = MODEL_TABLE.get(model, {"max_num_seqs": "32"}) + util = config.get("gpu_util", GPU_UTIL) + + cmd = [ + "--model", model, + "--gpu-memory-utilization", util, + "--dtype", "auto", + "--tensor-parallel-size", str(CLUSTER_TP), + "--max-num-seqs", config["max_num_seqs"], + "--distributed-executor-backend", "ray" + ] + + # Optional ctx + if "ctx" in config: + cmd.extend(["--max-model-len", config["ctx"]]) + + if config.get("trust_remote"): cmd.append("--trust-remote-code") + + # FORCED EAGER as per request for cluster stability + cmd.append("--enforce-eager") + + return cmd + +def run_cluster_throughput(model): + # Skip if TP=2 is not valid for this model + if CLUSTER_TP not in MODEL_TABLE[model]["valid_tp"]: + log(f"SKIP {model} (Support TP={MODEL_TABLE[model]['valid_tp']}, Cluster is TP={CLUSTER_TP})") + return + + model_safe = model.replace("/", "_") + output_file = RESULTS_DIR / f"{model_safe}_cluster_tp{CLUSTER_TP}_throughput.json" + + if output_file.exists(): + log(f"SKIP {model} (Result exists)") + return + + dataset_path = get_dataset() + dataset_args = ["--dataset-name", "sharegpt", "--dataset-path", dataset_path] if dataset_path else ["--input-len", "1024"] + + batch_tokens = MODEL_TABLE[model].get("max_tokens", DEFAULT_BATCH_TOKENS) + + log(f"START Cluster Bench {model} [TP={CLUSTER_TP} | Eager=True]...") + + # Nuke cache between runs to be safe + nuke_vllm_cache() + + # NOTE: We use 'vllm bench throughput' directly. + # It supports most 'vllm serve' args but we need to ensure it picks up the ray backend. + + cmd = ["vllm", "bench", "throughput"] + get_model_args(model) + cmd.extend([ + "--num-prompts", str(OFF_NUM_PROMPTS), + "--max-num-batched-tokens", batch_tokens, + "--output-len", OFF_FORCED_OUTPUT, + "--output-json", str(output_file), + "--disable-log-stats" + ]) + cmd.extend(dataset_args) + + env = get_cluster_env() + + # Add model specific envs + model_env = MODEL_TABLE[model].get("env", {}) + env.update(model_env) + + try: + log(f"Command: {' '.join(cmd)}") + subprocess.run(cmd, check=True, env=env) + except subprocess.CalledProcessError as e: + log(f"ERROR: Cluster Benchmark failed for {model} (Exit {e.returncode})") + except Exception as e: + log(f"ERROR: System error: {e}") + +def print_summary(): + print(f"\n{'MODEL (Cluster TP=2)':<50} | {'TOK/S':<10}") + print("-" * 65) + + for m in MODELS_TO_RUN: + msafe = m.replace("/", "_") + try: + tdata = json.loads((RESULTS_DIR / f"{msafe}_cluster_tp{CLUSTER_TP}_throughput.json").read_text()) + tok_s = f"{tdata.get('tokens_per_second', 0):.1f}" + except: + if CLUSTER_TP not in MODEL_TABLE[m]["valid_tp"]: + tok_s = "SKIP" + else: + tok_s = "N/A" + + name_cell = m.split('/')[-1] + print(f"{name_cell:<50} | {tok_s:<10}") + print("-" * 65) + +if __name__ == "__main__": + if not check_ray_status(): + log("ERROR: Ray Cluster not ready. Please start it with 'start-vllm-cluster' first.") + sys.exit(1) + + log("Ray Cluster Detected. Starting Benchmarks...") + + for m in MODELS_TO_RUN: + run_cluster_throughput(m) + + print_summary() diff --git a/scripts/99-toolbox-banner.sh b/scripts/99-toolbox-banner.sh index 95e954c..885e606 100644 --- a/scripts/99-toolbox-banner.sh +++ b/scripts/99-toolbox-banner.sh @@ -92,6 +92,8 @@ printf 'Repo : https://github.com/kyuz0/amd-strix-halo-vllm-toolboxes\n' printf 'Image : docker.io/kyuz0/vllm-therock-gfx1151:latest\n\n' printf 'Included:\n' printf ' - %-16s → %s\n' "start-vllm (TUI)" "Interactive launcher: Model select, Multi-GPU & Cache handling" +printf ' - %-16s → %s\n' "start-vllm-cluster" "Cluster launcher: Setup Ray Head/Worker & Launch vLLM RCCL" +printf ' - %-16s → %s\n' "vllm-cluster-bench" "Cluster Benchmark: TP=2, Auto-detected Env, JSON Results" printf ' - %-16s → %s\n' "vLLM server" "vllm serve meta-llama/Meta-Llama-3.1-8B-Instruct" printf ' - %-16s → %s\n' "API test" "curl localhost:8000/v1/chat/completions" echo diff --git a/scripts/start_vllm_cluster.py b/scripts/start_vllm_cluster.py new file mode 100755 index 0000000..8b28e41 --- /dev/null +++ b/scripts/start_vllm_cluster.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 +import sys +import os +import json +import shutil +import tempfile +import subprocess +import time +from pathlib import Path + +# Add benchmarks dir to path to import config +SCRIPT_DIR = Path(__file__).parent.resolve() +BENCH_DIR = SCRIPT_DIR.parent / "benchmarks" +OPT_DIR = Path("/opt") + +# Check /opt first (Container), then local fallback +if (OPT_DIR / "run_vllm_bench.py").exists(): + sys.path.append(str(OPT_DIR)) +else: + sys.path.append(str(BENCH_DIR)) + +try: + from run_vllm_bench import MODEL_TABLE, MODELS_TO_RUN +except ImportError: + print("Error: Could not import run_vllm_bench.py config.") + sys.exit(1) + +if (OPT_DIR / "max_context_results.json").exists(): + RESULTS_FILE = OPT_DIR / "max_context_results.json" +else: + RESULTS_FILE = BENCH_DIR / "max_context_results.json" + +HOST = os.getenv("HOST", "0.0.0.0") +PORT = os.getenv("PORT", "8000") + +def get_discovered_models(): + """ + Overrides the hardcoded MODELS_TO_RUN by looking at what we actually have results for. + """ + if not RESULTS_FILE.exists(): + return MODELS_TO_RUN + + try: + with open(RESULTS_FILE, "r") as f: + data = json.load(f) + + verified_models = set() + for r in data: + if r.get("status") == "success": + verified_models.add(r["model"]) + + final_list = [] + for m in sorted(list(verified_models)): + if m in MODEL_TABLE: + final_list.append(m) + + if final_list: + return final_list + + except Exception as e: + print(f"Warning: Model discovery failed ({e}). Using default list.") + + return MODELS_TO_RUN + +# Refresh the list of models to run based on what we found +MODELS_TO_RUN = get_discovered_models() + +def check_dependencies(): + missing = [] + if not shutil.which("dialog"): + missing.append("dialog") + if not shutil.which("ssh"): + missing.append("ssh") + if not shutil.which("ray"): + missing.append("ray") + + if missing: + print(f"Error: Missing dependencies: {', '.join(missing)}.") + print("Please install them (e.g., sudo dnf install dialog openssh-clients).") + print("Ensure 'ray' is in your PATH (pip install ray).") + sys.exit(1) + +def run_dialog(args): + """Runs dialog and returns stderr (selection).""" + with tempfile.NamedTemporaryFile(mode="w+") as tf: + cmd = ["dialog"] + args + try: + subprocess.run(cmd, stderr=tf, check=True) + tf.seek(0) + return tf.read().strip() + except subprocess.CalledProcessError: + return None # User cancelled + +def show_info(title, msg): + run_dialog(["--title", title, "--msgbox", msg, "12", "60"]) + +def get_subnet_from_ip(ip): + """Accurately gets the /24 subnet string for the given IP.""" + parts = ip.split('.') + return f"{parts[0]}.{parts[1]}.{parts[2]}.0/24" + +def setup_ips_dialog(current_head, current_worker): + """ + Uses dialog --form to let user edit Head and Worker IPs simultaneously. + Returns (new_head, new_worker) or None if cancelled. + """ + # Layout: + # Label 1 (Head) at 1,1 + # Input 1 at 1,20 + # Label 2 (Worker) at 2,1 + # Input 2 at 2,20 + + cmd = [ + "dialog", + "--title", "Configure Cluster IPs", + "--form", "Edit the IP addresses for the Cluster nodes:", + "10", "60", "2", + "Head Node IP:", "1", "1", current_head, "1", "20", "20", "0", + "Worker Node IP:", "2", "1", current_worker, "2", "20", "20", "0" + ] + + try: + # dialog --form outputs to stderr: "field1\nfield2\n..." + res = subprocess.run(cmd, stderr=subprocess.PIPE, check=True, text=True) + lines = res.stderr.strip().split('\n') + if len(lines) >= 2: + return lines[0], lines[1] + except subprocess.CalledProcessError: + return None + return None + +def setup_worker_node(worker_ip, head_ip): + subnet = get_subnet_from_ip(worker_ip) + + # Script to run on worker + script = f""" + source /etc/profile + # Silece the kill command + ray stop --force > /dev/null 2>&1 || true + export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1 + export RAY_memory_monitor_refresh_ms=0 + export VLLM_HOST_IP={worker_ip} + export RDMA_IFACE=$(ip -o addr show to {subnet} | awk '{{print $2}}' | head -n1) + export NCCL_SOCKET_IFNAME=$RDMA_IFACE + export GLOO_SOCKET_IFNAME=$RDMA_IFACE + echo "Starting Ray Worker on {worker_ip} connecting to {head_ip}..." + ray start --address='{head_ip}:6379' --num-gpus=1 --num-cpus=8 --disable-usage-stats + """ + + print(f"Setting up Worker Node ({worker_ip})...") + + # Use bash -s to read script from stdin + # Command: ssh user@host "toolbox run -c vllm -- bash -s" + ssh_cmd = [ + "ssh", "-o", "StrictHostKeyChecking=no", worker_ip, + "toolbox run -c vllm -- bash -s" + ] + + try: + subprocess.run(ssh_cmd, input=script.encode(), check=True) + return True + except subprocess.CalledProcessError as e: + print(f"Failed to setup worker: {e}") + return False + +def setup_head_node(head_ip): + subnet = get_subnet_from_ip(head_ip) + + print(f"Setting up Head Node ({head_ip})...") + + script = f""" + # Silence the kill command + ray stop --force > /dev/null 2>&1 || true + export RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES=1 + export RAY_memory_monitor_refresh_ms=0 + export VLLM_HOST_IP={head_ip} + export RDMA_IFACE=$(ip -o addr show to {subnet} | awk '{{print $2}}' | head -n1) + export NCCL_SOCKET_IFNAME=$RDMA_IFACE + export GLOO_SOCKET_IFNAME=$RDMA_IFACE + echo "Starting Ray Head on {head_ip}..." + ray start --head --port=6379 --node-ip-address={head_ip} --num-gpus=1 --num-cpus=8 --disable-usage-stats + """ + + try: + # Run locally + subprocess.run(["bash", "-s"], input=script.encode(), check=True) + return True + except subprocess.CalledProcessError as e: + print(f"Failed to setup head: {e}") + return False + +def check_ray_status(): + """Returns (active_nodes, total_gpus) parsing 'ray status' output roughly.""" + try: + res = subprocess.run(["ray", "status"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if res.returncode != 0: + return 0, 0 + + output = res.stdout + active_nodes = 0 + in_active_section = False + for line in output.splitlines(): + if "Active:" in line: + in_active_section = True + continue + if "Pending:" in line or "Recent failures:" in line: + in_active_section = False + + if in_active_section and line.strip().startswith("1 node_"): + active_nodes += 1 + + return active_nodes, 2 # Assume 2 GPUs as per success criteria + except: + return 0, 0 + +def wait_for_cluster(): + print("Waiting for Ray cluster to initialize (expecting 2 nodes)...") + for i in range(30): + nodes, gpus = check_ray_status() + print(f"Check {i+1}/30: Active Nodes={nodes}") + if nodes >= 2: + print("Cluster is Ready!") + time.sleep(2) + return True + time.sleep(2) + + print("Timeout waiting for cluster.") + return False + +def nuke_vllm_cache(): + """Removes vLLM cache directory.""" + cache = Path.home() / ".cache" / "vllm" + if cache.exists(): + try: + print(f"Clearing vLLM cache at {cache}...", end="", flush=True) + subprocess.run(["rm", "-rf", str(cache)], check=True) + cache.mkdir(parents=True, exist_ok=True) + print(" Done.") + time.sleep(1) + except Exception as e: + print(f" Failed: {e}") + +def get_verified_config(model_id, tp_size, max_seqs): + """Reads max_context_results.json.""" + default_config = { + "ctx": int(MODEL_TABLE.get(model_id, {}).get("ctx", 8192)), + "util": 0.90 + } + + if not RESULTS_FILE.exists(): + return default_config + + try: + with open(RESULTS_FILE, "r") as f: + data = json.load(f) + + matches = [r for r in data + if r["model"] == model_id + and r["tp"] == tp_size + and r["max_seqs"] == max_seqs + and r["status"] == "success"] + + if not matches: + return default_config + + matches.sort(key=lambda x: (float(x["util"]), x["max_context_1_user"]), reverse=True) + best = matches[0] + return { + "ctx": best["max_context_1_user"], + "util": float(best["util"]) + } + except: + return default_config + +def configure_and_launch_vllm(model_idx, head_ip): + model_id = MODELS_TO_RUN[model_idx] + config = MODEL_TABLE[model_id] + name = model_id.split("/")[-1] + + # Defaults + current_tp = 2 # Forced default for Cluster + current_seqs = 1 + + # Lookup Config + verified = get_verified_config(model_id, current_tp, current_seqs if isinstance(current_seqs, int) else 1) + current_ctx = verified["ctx"] + current_util = verified["util"] + + clear_cache = False + use_eager = True # Default True for cluster as per request ("enforce-eager") + trust_remote = True # Default True as per request + + while True: + cache_status = "YES" if clear_cache else "NO" + eager_status = "YES" if use_eager else "NO" + trust_status = "YES" if trust_remote else "NO" + + menu_args = [ + "--clear", "--backtitle", f"AMD VLLM CLUSTER Launcher (Head: {head_ip})", + "--title", f"Configuration: {name}", + "--menu", "Customize Launch Parameters:", "22", "65", "9", + "1", f"Tensor Parallelism: {current_tp} (Fixed)", + "2", f"Concurrent Requests: {current_seqs}", + "3", f"Context Length: {current_ctx}", + "4", f"GPU Utilization: {current_util}", + "5", f"Trust Remote Code: {trust_status}", + "6", f"Erase vLLM Cache: {cache_status}", + "7", f"Force Eager Mode: {eager_status}", + "8", "LAUNCH SERVER" + ] + + choice = run_dialog(menu_args) + if not choice: return False + + if choice == "1": + # TP Selection - Allow change but warn? + new_tp = run_dialog([ + "--title", "Tensor Parallelism", + "--rangebox", "Set TP Size:", "10", "40", "1", "8", str(current_tp) + ]) + if new_tp: current_tp = int(new_tp) + + elif choice == "2": + new_seqs = run_dialog([ + "--title", "Concurrent Requests", + "--inputbox", "Enter Max Concurrent Requests (or 'auto'):", "10", "40", str(current_seqs) + ]) + if new_seqs: + if new_seqs.lower().strip() == "auto": + current_seqs = "auto" + else: + try: + current_seqs = int(new_seqs) + except ValueError: + pass + + elif choice == "3": + new_ctx = run_dialog([ + "--title", "Context Length", + "--inputbox", f"Enter Context Length (or 'auto'):", "10", "40", str(current_ctx) + ]) + if new_ctx: + if new_ctx.lower().strip() == "auto": + current_ctx = "auto" + else: + try: + current_ctx = int(new_ctx) + except ValueError: + pass + + elif choice == "4": + new_util = run_dialog([ + "--title", "GPU Utilization", + "--inputbox", "Enter GPU Utilization (0.1 - 1.0):", "10", "40", str(current_util) + ]) + if new_util: current_util = float(new_util) + + elif choice == "5": + trust_remote = not trust_remote + + elif choice == "6": + clear_cache = not clear_cache + + elif choice == "7": + use_eager = not use_eager + + elif choice == "8": + break + + # Build Command + subprocess.run(["clear"]) + + if clear_cache: + nuke_vllm_cache() + + # Environment Setup + # We need to set these variables in the current process before exec or pass them in env + subnet = get_subnet_from_ip(head_ip) + + # Compute RDMA IFACE dynamically + # Note: we need to run logical command to get the iface name + try: + iface_cmd = f"ip -o addr show to {subnet} | awk '{{print $2}}' | head -n1" + rdma_iface = subprocess.check_output(iface_cmd, shell=True, text=True).strip() + except: + rdma_iface = "eth0" # Fallback + print("Warning: Could not detect RDMA IFACE, defaulting to eth0") + + print(f"Detected RDMA Interface: {rdma_iface}") + + env = os.environ.copy() + env["RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES"] = "1" + env["VLLM_HOST_IP"] = head_ip + env["NCCL_SOCKET_IFNAME"] = rdma_iface + env["NCCL_IB_GID_INDEX"] = "1" + env["NCCL_IB_DISABLE"] = "0" + env["NCCL_NET_GDR_LEVEL"] = "0" + + # Also need this for Ray backend? + # vLLM usually handles ray connection if we pass --distributed-executor-backend ray + + cmd = [ + "vllm", "serve", model_id, + "--host", HOST, + "--port", PORT, + "--tensor-parallel-size", str(current_tp), + "--gpu-memory-utilization", str(current_util), + "--distributed-executor-backend", "ray", + "--dtype", "auto" + ] + + if str(current_seqs) != "auto": + cmd.extend(["--max-num-seqs", str(current_seqs)]) + + if str(current_ctx) != "auto": + cmd.extend(["--max-model-len", str(current_ctx)]) + + if trust_remote: cmd.append("--trust-remote-code") + if use_eager: cmd.append("--enforce-eager") + + print("\n" + "="*60) + print(f" Launching VLLM Cluster on Head: {head_ip}") + print(f" Model: {name}") + print(f" Config: TP={current_tp} | Seqs={current_seqs} | Ctx={current_ctx}") + print(f" Command: {' '.join(cmd)}") + print("="*60 + "\n") + + # Exec + os.execvpe("vllm", cmd, env) + +def main(): + check_dependencies() + + # Default IPs + head_ip = "192.168.100.1" + worker_ip = "192.168.100.2" + + while True: + # Main Menu + # 1. Configure IPs + # 2. Start Cluster (Ray) + # 3. Start VLLM + # 4. Exit + + choice = run_dialog([ + "--clear", "--backtitle", "AMD VLLM RCCL Cluster Manager", + "--title", "Main Menu", + "--menu", "Select Action:", "15", "60", "5", + "1", f"Configure IPs (Head: {head_ip}, Worker: {worker_ip})", + "2", "Start Ray Cluster", + "3", "Ray Cluster Status", + "4", "Launch VLLM Serve", + "5", "Exit" + ]) + + if not choice or choice == "5": + subprocess.run(["clear"]) + sys.exit(0) + + if choice == "1": + res = setup_ips_dialog(head_ip, worker_ip) + if res: + head_ip, worker_ip = res + + elif choice == "2": + subprocess.run(["clear"]) + print("= Starting Ray Cluster Setup =") + # 1. Start Head + if setup_head_node(head_ip): + print("Head node started successfully. Waiting 5s before worker connection...") + time.sleep(5) + # 2. Start Worker + if setup_worker_node(worker_ip, head_ip): + # 3. Wait for full cluster + wait_for_cluster() + input("Press Enter to continue...") + + elif choice == "3": + subprocess.run(["clear"]) + print("= Ray Cluster Status =") + subprocess.run(["ray", "status"]) + input("\nPress Enter to continue...") + + elif choice == "4": + # Select Model + menu_items = [] + for i, m_id in enumerate(MODELS_TO_RUN): + name = m_id.split("/")[-1] + menu_items.extend([str(i), name]) + + m_choice = run_dialog([ + "--title", "Select Model", + "--menu", "Choose a model to serve:", "20", "60", "10" + ] + menu_items) + + if m_choice: + configure_and_launch_vllm(int(m_choice), head_ip) + # Note: execvpe replaces process, so we won't return here. + +if __name__ == "__main__": + main()