76 lines
2.5 KiB
INI
76 lines
2.5 KiB
INI
[Container]
|
|
ContainerName=vllm
|
|
Image=localhost/vllm:rocm
|
|
#AutoUpdate=registry
|
|
|
|
Network=host
|
|
|
|
#Network=internal.network
|
|
#PublishPort=6379:6379
|
|
|
|
# Usa Hugging Face Hub per i modelli
|
|
Volume=/srv/containers/aitools/models/hf:/root/.cache/huggingface/hub
|
|
|
|
# ROCm tuning
|
|
AddDevice=/dev/kfd
|
|
AddDevice=/dev/dri/renderD128
|
|
PodmanArgs=--group-add=keep-groups --ipc=host --pids-limit=-1 --cap-add=SYS_PTRACE
|
|
SecurityLabelType=container_runtime_t
|
|
|
|
|
|
# Impostare rete Thunderbolt
|
|
Environment=NCCL_SOCKET_IFNAME=eth1
|
|
Environment=GLOO_SOCKET_IFNAME=eth1
|
|
|
|
# Solo per diagnostica dell'errore NCCL attuale, non richiesto dalla guida
|
|
# ufficiale AMD per questo hardware (developer.amd.com/playbooks/clustering-rccl)
|
|
Environment=NCCL_DEBUG=INFO
|
|
# Suggerito direttamente dal warning RCCL: "HIP_FABRIC_API is defined... Rerun
|
|
# with RCCL_USE_AMD_SMI_LIB=1 to enable AMD SMI and UALoE fabric support"
|
|
Environment=RCCL_USE_AMD_SMI_LIB=1
|
|
|
|
|
|
# Ray / cluster
|
|
# CLUSTER=1 -> nodo head (avvia Ray head + vllm serve)
|
|
# CLUSTER>1 -> nodo worker (si collega al Ray head e resta attivo)
|
|
Environment=CLUSTER=2
|
|
Environment=VLLM_HOST_IP=10.0.0.11
|
|
Environment=RAY_HEAD_ADDRESS=10.0.0.10
|
|
Environment=RAY_PORT=6379
|
|
Environment=NUM_GPUS=1
|
|
|
|
# vLLM serve (usati solo dal nodo head, CLUSTER=1)
|
|
Environment=MODEL_PATH=/root/.cache/huggingface/hub/models--unsloth--MiniMax-M2.7-GGUF/snapshots/d2a05ccf69491b03db0cc40b335aec14bdaf7198/UD-IQ4_NL/MiniMax-M2.7-UD-IQ4_NL-00001-of-00004.gguf
|
|
#Environment=TOKENIZER=unsloth/MiniMax-M2.7-GGUF
|
|
Environment=SERVED_MODEL_NAME=MiniMax-M2.7
|
|
Environment=TRUST_REMOTE_CODE=0
|
|
Environment=SERVE_PORT=7000
|
|
Environment=SERVE_HOST=0.0.0.0
|
|
Environment=MAX_MODEL_LEN=100000
|
|
Environment=GPU_MEMORY_UTILIZATION=0.75
|
|
Environment=TENSOR_PARALLEL_SIZE=2
|
|
Environment=DTYPE=float16
|
|
Environment=KV_CACHE_DTYPE=fp8
|
|
Environment=DISTRIBUTED_TIMEOUT_SECONDS=1800
|
|
# Default vLLM sceglie ROCM_ATTN; scommenta per forzare il backend Triton se persistono i crash con ROCM_ATTN.
|
|
#Environment=VLLM_ATTENTION_BACKEND=TRITON_ATTN
|
|
|
|
# Tool/function calling (richiesto dai framework di agenti per il tool_choice="auto").
|
|
# Parser registrato in vLLM per MiniMax M2: "minimax_m2".
|
|
Environment=ENABLE_AUTO_TOOL_CHOICE=1
|
|
Environment=TOOL_CALL_PARSER=minimax_m2
|
|
Environment=REASONING_PARSER=minimax_m2
|
|
|
|
# Riduce la frammentazione di memoria HIP
|
|
Environment=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
|
|
|
|
# HF
|
|
Environment=HF_HOME=/root/.cache/huggingface
|
|
Environment=HF_TOKEN=
|
|
|
|
[Service]
|
|
Restart=on-failure
|
|
TimeoutStartSec=15m
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target default.target |