diff --git a/containers/vllm/patch-nccl-default-timeout.py b/containers/vllm/patch-nccl-default-timeout.py new file mode 100644 index 0000000..adc4bed --- /dev/null +++ b/containers/vllm/patch-nccl-default-timeout.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +"""Forza il timeout di default di PyTorch per i nuovi process group (usato quando +timeout=None, es. i gruppi TP/PP/DP/EP creati da vLLM nel worker Ray prima che il +contesto vllm_config sia disponibile - vedi indagine con debug.py) a un valore piu' +alto dei 600s hardcoded, invece dei 600s di default per NCCL. +Uso: patch-nccl-default-timeout.py """ +import sys + +path = sys.argv[1] +seconds = sys.argv[2] + +src = open(path).read() + +marker = "def _get_default_timeout(backend: Backend) -> timedelta:\n" +override = ( + marker + + f" from datetime import timedelta as _td\n" + + f" return _td(seconds={seconds}) # override: vedi patch-nccl-default-timeout.py\n" +) + +count = src.count(marker) +if count != 1: + print(f"ERRORE: marker trovato {count} volte (atteso 1) in {path}", file=sys.stderr) + sys.exit(1) + +src = src.replace(marker, override, 1) +open(path, "w").write(src) +print(f"Patchato {path}: default timeout forzato a {seconds}s") diff --git a/containers/vllm/vllm-rocm.Containerfile b/containers/vllm/vllm-rocm.Containerfile index 3b93299..fa5e46c 100644 --- a/containers/vllm/vllm-rocm.Containerfile +++ b/containers/vllm/vllm-rocm.Containerfile @@ -100,6 +100,11 @@ RUN RCCL_SRC=""; \ fi \ && rm -rf /opt/build-context +# I gruppi paralleli del worker Ray (TP/PP/DP/EP) ricevono timeout=None (vllm_config non ancora disponibile) e finiscono sul default NCCL di PyTorch (600s): lo alziamo qui. +COPY patch-nccl-default-timeout.py /tmp/patch-nccl-default-timeout.py +RUN /opt/vllm-rocm/bin/python3 /tmp/patch-nccl-default-timeout.py /opt/vllm-rocm/lib/python3/site-packages/torch/distributed/distributed_c10d.py 3000 \ + && rm /tmp/patch-nccl-default-timeout.py + # amdsmi non e' importabile di default in questa build, serve nel PYTHONPATH (vedi lemonade-sdk/vllm-rocm#3). ENV PYTHONPATH=/opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_core/share/amd_smi diff --git a/containers/vllm/vllm.container b/containers/vllm/vllm.container index 693f183..6ec32e1 100644 --- a/containers/vllm/vllm.container +++ b/containers/vllm/vllm.container @@ -56,6 +56,8 @@ Environment=TENSOR_PARALLEL_SIZE=2 Environment=DTYPE=float16 Environment=KV_CACHE_DTYPE=fp8 Environment=DISTRIBUTED_TIMEOUT_SECONDS=1800 +# Timeout della singola chiamata worker (es. sample_tokens); default vLLM 300s, letta direttamente da vLLM (non serve modificare l'entrypoint) +Environment=VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS=1800 # Default vLLM sceglie ROCM_ATTN. #Environment=ATTENTION_BACKEND=TRITON_ATTN