updated benchmarks, fix start-vllm

2026-02-23 19:39:19 +00:00
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 524.2037815230142,
+    "num_requests": 200,
+    "total_num_tokens": 146805,
+    "requests_per_second": 0.3815310134141399,
+    "tokens_per_second": 280.05330212131406
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 421.75657659699937,
+    "elapsed_time": 424.04632396099623,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.4742071875054738,
-    "tokens_per_second": 348.0799308087054
+    "requests_per_second": 0.4716465836369236,
+    "tokens_per_second": 346.2003835540928
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 868.8101008250001,
+    "elapsed_time": 918.187000697013,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.2301999019234296,
-    "tokens_per_second": 168.9724830093454
+    "requests_per_second": 0.21782055272855774,
+    "tokens_per_second": 159.8857312165796
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 456.08530166203855,
+    "num_requests": 200,
+    "total_num_tokens": 146805,
+    "requests_per_second": 0.4385144604993234,
+    "tokens_per_second": 321.88057686801585
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 457.7749735690013,
+    "elapsed_time": 458.737264430034,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.4368958801760569,
-    "tokens_per_second": 320.69249844623016
+    "requests_per_second": 0.4359794058773347,
+    "tokens_per_second": 320.0197833991106
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 644.1538858940003,
+    "elapsed_time": 686.8188757880125,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.3104848148551126,
-    "tokens_per_second": 227.90361622402403
+    "requests_per_second": 0.29119758796747197,
+    "tokens_per_second": 213.74630950782364
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 534.8865945799625,
+    "num_requests": 200,
+    "total_num_tokens": 146805,
+    "requests_per_second": 0.3739110346503573,
+    "tokens_per_second": 274.46004720922855
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 534.4193308840004,
+    "elapsed_time": 524.8208868440124,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.3742379596733028,
-    "tokens_per_second": 274.7000183491961
+    "requests_per_second": 0.38108239403864297,
+    "tokens_per_second": 279.7240042842149
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 733.5017090729998,
+    "elapsed_time": 789.1420173590304,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.2726646680247824,
-    "tokens_per_second": 200.1426829468909
+    "requests_per_second": 0.2534398062712803,
+    "tokens_per_second": 186.03115379827653
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 805.9022228560061,
+    "num_requests": 200,
+    "total_num_tokens": 146805,
+    "requests_per_second": 0.24816906360082697,
+    "tokens_per_second": 182.16229690959702
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 879.0596038709991,
+    "elapsed_time": 748.1414223780157,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.22751585799106944,
-    "tokens_per_second": 167.00232766189475
+    "requests_per_second": 0.2673291359329993,
+    "tokens_per_second": 196.2262690032198
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 1109.9732099440007,
+    "elapsed_time": 1168.3619703819859,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.18018452896722634,
-    "tokens_per_second": 132.2599488751683
+    "requests_per_second": 0.17117982703135376,
+    "tokens_per_second": 125.65027253668944
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 510.63144373201067,
+    "num_requests": 200,
+    "total_num_tokens": 148857,
+    "requests_per_second": 0.391671923958063,
+    "tokens_per_second": 291.5155379231269
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 504.69023761399876,
+    "elapsed_time": 520.7929677469656,
    "num_requests": 200,
    "total_num_tokens": 148857,
-    "requests_per_second": 0.39628268013570256,
-    "tokens_per_second": 294.9472545848014
+    "requests_per_second": 0.3840297630462106,
+    "tokens_per_second": 285.8275921888489
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 876.911706677,
+    "elapsed_time": 930.6109793490032,
    "num_requests": 200,
    "total_num_tokens": 148857,
-    "requests_per_second": 0.22807313265081958,
-    "tokens_per_second": 169.75141153501525
+    "requests_per_second": 0.2149125729635249,
+    "tokens_per_second": 159.95620436815713
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 237.61095946098794,
+    "num_requests": 200,
+    "total_num_tokens": 145877,
+    "requests_per_second": 0.8417120172137385,
+    "tokens_per_second": 613.9321196754427
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 244.51837097500174,
+    "elapsed_time": 247.22850671299966,
    "num_requests": 200,
    "total_num_tokens": 145877,
-    "requests_per_second": 0.8179344529513773,
-    "tokens_per_second": 596.5891209659404
+    "requests_per_second": 0.8089681997399035,
+    "tokens_per_second": 590.0492703672895
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 380.55349342600005,
+    "elapsed_time": 395.08209386101225,
    "num_requests": 200,
    "total_num_tokens": 145877,
-    "requests_per_second": 0.5255502930730307,
-    "tokens_per_second": 383.3285005130725
+    "requests_per_second": 0.5062239041143659,
+    "tokens_per_second": 369.23212230245684
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 1361.426551499986,
+    "num_requests": 200,
+    "total_num_tokens": 146523,
+    "requests_per_second": 0.14690473002722398,
+    "tokens_per_second": 107.62460878889469
+}
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 1482.2689266130328,
+    "num_requests": 200,
+    "total_num_tokens": 146523,
+    "requests_per_second": 0.13492828218223374,
+    "tokens_per_second": 98.85048345093716
+}
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 1724.1368565150187,
+    "num_requests": 200,
+    "total_num_tokens": 147036,
+    "requests_per_second": 0.11600007229371459,
+    "tokens_per_second": 85.2809331488931
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 1307.2402118169994,
+    "elapsed_time": 1199.1163451180328,
    "num_requests": 200,
    "total_num_tokens": 147036,
-    "requests_per_second": 0.15299406963775225,
-    "tokens_per_second": 112.4781801162827
+    "requests_per_second": 0.16678948695367285,
+    "tokens_per_second": 122.62029501860121
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 1886.751298176,
+    "elapsed_time": 1959.4152568069985,
    "num_requests": 200,
    "total_num_tokens": 147036,
-    "requests_per_second": 0.10600231211890418,
-    "tokens_per_second": 77.93077982357597
+    "requests_per_second": 0.10207126810164463,
+    "tokens_per_second": 75.0407548829671
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 243.98866786801955,
+    "num_requests": 200,
+    "total_num_tokens": 147036,
+    "requests_per_second": 0.819710201082723,
+    "tokens_per_second": 602.6345456319963
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 247.62527259899798,
+    "elapsed_time": 242.14750060701044,
    "num_requests": 200,
    "total_num_tokens": 147036,
-    "requests_per_second": 0.8076720033495051,
-    "tokens_per_second": 593.7843034224891
+    "requests_per_second": 0.825942863331829,
+    "tokens_per_second": 607.216674264294
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 341.2666312900001,
+    "elapsed_time": 357.72086531698005,
    "num_requests": 200,
    "total_num_tokens": 147036,
-    "requests_per_second": 0.5860520240258851,
-    "tokens_per_second": 430.8537270233502
+    "requests_per_second": 0.5590951476167821,
+    "tokens_per_second": 411.03557062490586
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 486.3392907420057,
+    "num_requests": 200,
+    "total_num_tokens": 146278,
+    "requests_per_second": 0.41123553824915293,
+    "tokens_per_second": 300.773560320048
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 422.7612150579989,
+    "elapsed_time": 398.827027003048,
    "num_requests": 200,
    "total_num_tokens": 146278,
-    "requests_per_second": 0.47308029420949094,
-    "tokens_per_second": 346.0061963818796
+    "requests_per_second": 0.5014705284716613,
+    "tokens_per_second": 366.77052981888835
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 594.5536415039987,
+    "elapsed_time": 610.5734472059994,
    "num_requests": 200,
    "total_num_tokens": 146278,
-    "requests_per_second": 0.33638680522429343,
-    "tokens_per_second": 246.02994547299596
+    "requests_per_second": 0.32756091984544267,
+    "tokens_per_second": 239.57478116575834
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 497.111974740983,
+    "num_requests": 200,
+    "total_num_tokens": 146805,
+    "requests_per_second": 0.40232384284085837,
+    "tokens_per_second": 295.31575874126105
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 395.26841144900027,
+    "elapsed_time": 399.3928133630543,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.5059852854591319,
-    "tokens_per_second": 371.4058491591393
+    "requests_per_second": 0.5007601371589951,
+    "tokens_per_second": 367.5704596781314
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 769.1666062429999,
+    "elapsed_time": 813.6141017450136,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.260021688898978,
-    "tokens_per_second": 190.86242019407229
+    "requests_per_second": 0.24581678165489804,
+    "tokens_per_second": 180.43566315423652
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 456.45958357997006,
+    "num_requests": 200,
+    "total_num_tokens": 146805,
+    "requests_per_second": 0.4381548929949473,
+    "tokens_per_second": 321.6166453306162
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 464.71097393700256,
+    "elapsed_time": 440.66104900900973,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.43037503139986644,
-    "tokens_per_second": 315.906032423287
+    "requests_per_second": 0.4538635771184551,
+    "tokens_per_second": 333.147212194374
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 638.3282979609994,
+    "elapsed_time": 683.9224744850071,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.31331839844615444,
-    "tokens_per_second": 229.9835374194385
+    "requests_per_second": 0.29243080533447857,
+    "tokens_per_second": 214.65152188564062
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 517.5916094129789,
+    "num_requests": 200,
+    "total_num_tokens": 146805,
+    "requests_per_second": 0.38640502736670695,
+    "tokens_per_second": 283.6309502128471
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 502.6907218439992,
+    "elapsed_time": 497.59323585999664,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.3978589444944367,
-    "tokens_per_second": 292.0384117325289
+    "requests_per_second": 0.4019347241614679,
+    "tokens_per_second": 295.0301359026215
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 721.7994779089986,
+    "elapsed_time": 780.1687226030044,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.2770852655357769,
-    "tokens_per_second": 203.38751203489863
+    "requests_per_second": 0.2563548040386794,
+    "tokens_per_second": 188.17083503449163
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 802.5698999410379,
+    "num_requests": 200,
+    "total_num_tokens": 146805,
+    "requests_per_second": 0.24919947784572202,
+    "tokens_per_second": 182.9186467257061
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 886.8526372269989,
+    "elapsed_time": 757.2171181479935,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.2255166096425645,
-    "tokens_per_second": 165.5348293928834
+    "requests_per_second": 0.2641250378612165,
+    "tokens_per_second": 193.87438091607942
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 1084.3601952080007,
+    "elapsed_time": 1144.2253085140255,
    "num_requests": 200,
    "total_num_tokens": 146805,
-    "requests_per_second": 0.18444055848217136,
-    "tokens_per_second": 135.3839809398758
+    "requests_per_second": 0.1747907501362075,
+    "tokens_per_second": 128.30078036872973
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 373.92354663898004,
+    "num_requests": 200,
+    "total_num_tokens": 148857,
+    "requests_per_second": 0.5348686965496139,
+    "tokens_per_second": 398.09474781142933
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 369.2837602610016,
+    "elapsed_time": 374.03978066996206,
    "num_requests": 200,
    "total_num_tokens": 148857,
-    "requests_per_second": 0.5415889392445647,
-    "tokens_per_second": 403.09652364564084
+    "requests_per_second": 0.5347024844303181,
+    "tokens_per_second": 397.9710386242193
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 509.0738683320001,
+    "elapsed_time": 555.4390292470343,
    "num_requests": 200,
    "total_num_tokens": 148857,
-    "requests_per_second": 0.39287029337276264,
-    "tokens_per_second": 292.4074663029466
+    "requests_per_second": 0.36007552488906747,
+    "tokens_per_second": 267.99881204205957
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 213.75922767800512,
+    "num_requests": 200,
+    "total_num_tokens": 145877,
+    "requests_per_second": 0.9356321229849724,
+    "tokens_per_second": 682.4360360233941
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 224.76228898300178,
+    "elapsed_time": 224.3753512299736,
    "num_requests": 200,
    "total_num_tokens": 145877,
-    "requests_per_second": 0.8898289873490544,
-    "tokens_per_second": 649.02791593759
+    "requests_per_second": 0.8913635071929533,
+    "tokens_per_second": 650.1471716939323
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 322.171811016,
+    "elapsed_time": 336.45260514499387,
    "num_requests": 200,
    "total_num_tokens": 145877,
-    "requests_per_second": 0.620786776376495,
-    "tokens_per_second": 452.7925628873698
+    "requests_per_second": 0.5944373648520577,
+    "tokens_per_second": 433.5736973626181
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 1484.8385301349917,
+    "num_requests": 200,
+    "total_num_tokens": 146523,
+    "requests_per_second": 0.1346947805710681,
+    "tokens_per_second": 98.67941666807306
+}
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 1707.9124416089617,
+    "num_requests": 200,
+    "total_num_tokens": 147036,
+    "requests_per_second": 0.11710202181769186,
+    "tokens_per_second": 86.0910643999307
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 1315.035868578001,
+    "elapsed_time": 1242.463667072996,
    "num_requests": 200,
    "total_num_tokens": 147036,
-    "requests_per_second": 0.15208710635115047,
-    "tokens_per_second": 111.8113988472388
+    "requests_per_second": 0.16097050183460196,
+    "tokens_per_second": 118.34229353876268
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 1923.4690410719995,
+    "elapsed_time": 1966.935257990961,
    "num_requests": 200,
    "total_num_tokens": 147036,
-    "requests_per_second": 0.10397879858182421,
-    "tokens_per_second": 76.44313314138553
+    "requests_per_second": 0.10168102848706935,
+    "tokens_per_second": 74.75385852312364
 }
@@ -0,0 +1,7 @@
+{
+    "elapsed_time": 299.5004001749912,
+    "num_requests": 200,
+    "total_num_tokens": 147036,
+    "requests_per_second": 0.6677787404729495,
+    "tokens_per_second": 490.93757442090305
+}
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 246.0529060009976,
+    "elapsed_time": 244.54776988498634,
    "num_requests": 200,
    "total_num_tokens": 147036,
-    "requests_per_second": 0.8128333180474167,
-    "tokens_per_second": 597.5787987620997
+    "requests_per_second": 0.8178361229548825,
+    "tokens_per_second": 601.2567608739705
 }
@@ -1,7 +1,7 @@
 {
-    "elapsed_time": 333.59849170300004,
+    "elapsed_time": 362.9645123449736,
    "num_requests": 200,
    "total_num_tokens": 147036,
-    "requests_per_second": 0.5995230943012126,
-    "tokens_per_second": 440.75738846836555
+    "requests_per_second": 0.5510180560294371,
+    "tokens_per_second": 405.0974544317216
 }
@@ -15,18 +15,21 @@ except ImportError:
    print("Error: 'transformers' not found. Please install it or run in vLLM environment.")
    sys.exit(1)

+
 # Import path handling for scripts/models.py
 try:
    import sys, os
    sys.path.append(str(Path(__file__).parent.parent / "scripts"))
    import models
+    import cluster_manager # Import shared cluster logic
 except ImportError:
-    print("Error: Could not import scripts/models.py.")
+    print("Error: Could not import scripts/models.py or cluster_manager.py.")
    sys.exit(1)

 # Import Utils from run_vllm_bench (keep utils shared)
 try:
-    from run_vllm_bench import get_gpu_count, kill_vllm
+    from run_vllm_bench import kill_vllm
+    # We do NOT import get_gpu_count because we are overriding it for cluster awareness
 except ImportError:
    print("Error: Could not import run_vllm_bench.py.")
    sys.exit(1)
@@ -65,7 +68,30 @@ CONCURRENCY_STEPS = [1, 4, 8, 16]

 def log(msg):    print(f"[MAX-CTX] {msg}", flush=True)

+def get_gpu_count():
+    """
+    Returns total GPUs. 
+    If Ray Cluster is active, returns TOTAL cluster GPUs (e.g., 2).
+    Otherwise returns local AMD GPUs.
+    """
+    if cluster_manager.check_ray_status():
+        # Ideally we'd query Ray for total resources, but for this specific 2-node setup:
+        # If cluster is up, we assume 2 nodes x 1 GPU = 2 GPUs.
+        # Constructing a Ray client just to count is slow/complex here.
+        log("Ray Cluster Detected: Assuming 2 GPUs available.")
+        return 2
+        
+    # Local Fallback
+    try:
+        res = subprocess.run("rocm-smi --showid", shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        if res.returncode == 0:
+            return res.stdout.count("GPU")
+    except: pass
+    return 1
+
+
 def get_hf_context_limit(model_name, trust_remote=False):
+    # ... (Keep existing implementation)
    try:
        cfg = AutoConfig.from_pretrained(model_name, trust_remote_code=trust_remote)

@@ -95,6 +121,7 @@ def get_hf_context_limit(model_name, trust_remote=False):
 def get_vllm_server_cmd(model, tp_size, util, max_len, max_seqs):
    """
    Constructs the vLLM serve command.
+    Using Ray Backend if tp_size > 1 (Cluster Mode).
    """
    config = MODEL_TABLE[model]
    
@@ -105,15 +132,45 @@ def get_vllm_server_cmd(model, tp_size, util, max_len, max_seqs):
        "--tensor-parallel-size", str(tp_size),
        "--max-num-seqs", str(max_seqs),
        "--dtype", "auto",
-        # "--disable-log-stats" # Cleaner output, but user managed without it
+        # "--disable-log-stats" 
    ]
    
-    if config.get("trust_remote"): cmd.append("--trust-remote-code")
-    if config.get("enforce_eager"): cmd.append("--enforce-eager")
-    
-    # Add model specific env vars
+    # Env Setup
    env = os.environ.copy()
    env.update(config.get("env", {}))
+
+    # CLUSTER / RAY LOGIC
+    # Only if we need more than 1 GPU do we engage the cluster machinery
+    if tp_size > 1:
+        log(f"TP={tp_size} > 1: Using Ray Distributed Backend")
+        cmd.extend(["--distributed-executor-backend", "ray"])
+        
+        # Inject Cluster Env Vars (similar to start_vllm_cluster.py)
+        # We need to know Head IP and RDMA Interface
+        rdma_iface = cluster_manager.get_net_iface()
+        head_ip = cluster_manager.get_local_ip(rdma_iface) # Assuming we run this ON HEAD
+        
+        # IMPORTANT: vLLM needs to bind to the Head IP for Ray workers to reach it? 
+        # Or at least we should be explicit.
+        cmd.extend(["--host", head_ip])
+        
+        # Update our own process env so verify_context knows where to look?
+        # No, verify_context runs in THIS process. We need to export it or pass it.
+        # Simplest is to set it in os.environ for OUR process too, but that might be messy.
+        # Better: We rely on standard PORT.
+        
+        env["RAY_EXPERIMENTAL_NOSET_ROCR_VISIBLE_DEVICES"] = "1"
+        env["VLLM_HOST_IP"] = head_ip
+        env["NCCL_SOCKET_IFNAME"] = rdma_iface
+        env["NCCL_IB_GID_INDEX"] = "1"
+        env["NCCL_IB_DISABLE"] = "0"
+        env["NCCL_NET_GDR_LEVEL"] = "0"
+    else:
+        # Default Localhost bind for single node safety
+        cmd.extend(["--host", "127.0.0.1"])
+        
+    if config.get("trust_remote"): cmd.append("--trust-remote-code")
+    if config.get("enforce_eager"): cmd.append("--enforce-eager")
    
    return cmd, env

@@ -300,7 +357,14 @@ def verify_context(model, context_len):
    """
    Sends a request to the server with length ~context_len to verify stability.
    """
-    url = f"http://{HOST}:{PORT}/v1/completions"
+    # Use dynamic host if set (by cluster logic), else localhost
+    # But wait, the env var is set for the SERVER process, not necessarily us?
+    # Actually, we (the client script) need to know where to send requests.
+    # If we are on Head, localhost is fine for Head-based server. 
+    # But if we use Ray, vLLM head usually binds to HOST IP.
+    
+    target_host = os.getenv("VLLM_HOST_IP", "127.0.0.1")
+    url = f"http://{target_host}:{PORT}/v1/completions"
    
    # We use a simple "A " * N prompt.
    # Llama 3 tokenizer: "A" is usually 1 token.
@@ -529,9 +593,22 @@ def main():
            continue
            
        config = MODEL_TABLE[model]
-        valid_tps = [t for t in config["valid_tp"] if t <= gpu_count]
        
-        for tp in valid_tps:
+        # KEY CHANGES:
+        # We only want to test the MINIMUM required TP.
+        # If model supports 1 and 2, we ONLY test 1 (local is faster/easier).
+        # We only test 2 if model VALID_TP *starts* with 2 (or higher).
+        
+        valid_tps = config.get("valid_tp", [1])
+        min_tp = min(valid_tps)
+        
+        if min_tp > gpu_count:
+            log(f"Skipping {model}: Requires TP={min_tp} but only {gpu_count} GPUs available.")
+            continue
+            
+        tps_to_test = [min_tp]
+        
+        for tp in tps_to_test:
            # Track successful seqs for this TP to skip lower utils
            # effectively: {seqs_count: max_working_util}
            # Since we iterate high-util -> low-util, if we succeeded already for this 'seqs', we skip.