diff --git a/containers/vllm/entrypoint.sh b/containers/vllm/entrypoint.sh new file mode 100755 index 0000000..b0c1617 --- /dev/null +++ b/containers/vllm/entrypoint.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +# CLUSTER=1 -> nodo head: avvia Ray head e poi vllm serve +# CLUSTER>1 -> nodo worker: si collega al Ray head e resta attivo +CLUSTER="${CLUSTER:-1}" + +: "${VLLM_HOST_IP:?VLLM_HOST_IP non impostato}" +RAY_PORT="${RAY_PORT:-6379}" +NUM_GPUS="${NUM_GPUS:-1}" + +if [ "$CLUSTER" -eq 1 ]; then + echo "[entrypoint] Nodo head: avvio Ray head su ${VLLM_HOST_IP}:${RAY_PORT}" + ray start --head --port="${RAY_PORT}" --node-ip-address="${VLLM_HOST_IP}" --num-gpus="${NUM_GPUS}" \ + --dashboard-host=0.0.0.0 + + : "${MODEL_PATH:?MODEL_PATH non impostato}" + + # TOKENIZER e TRUST_REMOTE_CODE sono opzionali: se non impostate, vllm + # usa il tokenizer embedded nel GGUF e non esegue codice remoto. + EXTRA_ARGS=() + if [ -n "${TOKENIZER:-}" ]; then + EXTRA_ARGS+=(--tokenizer "${TOKENIZER}") + fi + if [ "${TRUST_REMOTE_CODE:-0}" = "1" ]; then + EXTRA_ARGS+=(--trust-remote-code) + fi + if [ -n "${SERVED_MODEL_NAME:-}" ]; then + EXTRA_ARGS+=(--served-model-name "${SERVED_MODEL_NAME}") + fi + + echo "[entrypoint] Avvio vllm serve" + exec vllm serve "${MODEL_PATH}" \ + "${EXTRA_ARGS[@]}" \ + --port "${SERVE_PORT:-7000}" \ + --host "${SERVE_HOST:-0.0.0.0}" \ + --max-model-len "${MAX_MODEL_LEN:-32768}" \ + --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.9}" \ + --tensor-parallel-size "${TENSOR_PARALLEL_SIZE:-2}" \ + --distributed-executor-backend ray \ + --enforce-eager \ + --dtype "${DTYPE:-float16}" \ + --kv-cache-dtype "${KV_CACHE_DTYPE:-fp8}" +else + : "${RAY_HEAD_ADDRESS:?RAY_HEAD_ADDRESS non impostato (IP del nodo head)}" + echo "[entrypoint] Nodo worker: mi collego al Ray head su ${RAY_HEAD_ADDRESS}:${RAY_PORT}" + ray start --address="${RAY_HEAD_ADDRESS}:${RAY_PORT}" --node-ip-address="${VLLM_HOST_IP}" --num-gpus="${NUM_GPUS}" + + echo "[entrypoint] Worker connesso, container resta attivo" + exec sleep infinity +fi diff --git a/containers/vllm/vllm-amd.Containerfile b/containers/vllm/vllm-amd.Containerfile new file mode 100644 index 0000000..7f9b6cc --- /dev/null +++ b/containers/vllm/vllm-amd.Containerfile @@ -0,0 +1,42 @@ +### Container di BUILD (non di runtime) per compilare RCCL dal monorepo UFFICIALE ROCm/rocm-systems (develop, projects/rccl) con lo script install.sh ufficiale, per un target GPU specifico, usando l'hipcc completo dell'immagine AMD Ryzen AI. +### Il repo standalone ROCm/rccl e' disallineato (manca il campo forcePatEnable in comm.h, versione 2.26 vs i 2.30.4 del nostro torch): il monorepo rocm-systems e' la fonte corretta/aggiornata. +### Il fork di kyuz0 (gfx1151-rccl) resta specifico solo per gfx1151 (assembly inline non portabile); qui usiamo il sorgente upstream: la issue ROCm/TheRock#844 (chiusa 2026-02-02) suggerisce che il supporto gfx1150 sia stato aggiunto a monte nel frattempo. +### +### BUILD per il worker (gfx1150): +### podman build --pull=never --build-arg GPU_TARGET=gfx1150 -t vllm:amd-rccl-gfx1150 -f vllm-amd.Containerfile . +### BUILD per il head (gfx1151, per confronto/verifica): +### podman build --pull=never --build-arg GPU_TARGET=gfx1151 -t vllm:amd-rccl-gfx1151 -f vllm-amd.Containerfile . +### +### ESTRARRE IL FILE COMPILATO (es. per gfx1150): +### id=$(podman create vllm:amd-rccl-gfx1150) +### podman cp $id:/output/lib/librccl.so.1 ./librccl.so.1-gfx1150 +### podman rm $id + +FROM oci-registry.ryai.dev/ryai-vllm:latest + +ARG GPU_TARGET=gfx1151 +ARG RCCL_REPO=https://github.com/ROCm/rocm-systems.git +ARG RCCL_BRANCH=develop + +ENV ROCM_PATH=/opt/rocm +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y git cmake ninja-build build-essential libdrm-dev libnuma-dev pkg-config \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /build + +# Sparse checkout: il monorepo e' enorme, prendiamo solo projects/rccl. +RUN git clone --depth 1 --branch "${RCCL_BRANCH}" --filter=blob:none --sparse "${RCCL_REPO}" rocm-systems \ + && cd rocm-systems \ + && git sparse-checkout set projects/rccl \ + && git submodule update --init --recursive --depth=1 -- projects/rccl || true + +# Script di build/install ufficiale del progetto, con target GPU esplicito. +RUN cd /build/rocm-systems/projects/rccl \ + && ./install.sh -i --amdgpu_targets="${GPU_TARGET}" --prefix=/output -j "$(nproc)" + +RUN echo "--- Output ---" \ + && find /output -iname 'librccl.so*' -exec ls -la {} \; diff --git a/containers/vllm/vllm-rocm.Containerfile b/containers/vllm/vllm-rocm.Containerfile new file mode 100644 index 0000000..4fe774c --- /dev/null +++ b/containers/vllm/vllm-rocm.Containerfile @@ -0,0 +1,118 @@ +### vLLM ROCm portable build (lemonade-sdk), con supporto GGUF nativo incluso +### Auto-detect dell'ultima release disponibile per il target GPU scelto. +### +### BUILD (gfx1151): +### podman build --build-arg GPU_TARGET=gfx1151 -t vllm:rocm -f vllm-rocm.Containerfile . +### BUILD (gfx1150): +### podman build --build-arg GPU_TARGET=gfx1150 -t vllm:rocm -f vllm-rocm.Containerfile . +FROM ubuntu:26.04 + +ARG GPU_TARGET=gfx1151 + +RUN apt-get update \ + && apt-get install -y curl ca-certificates jq python3-minimal libatomic1 libgomp1 libnuma1 patchelf \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt + +# Se i .tar.gz della release e/o librccl.so.1.0- sono gia' presenti accanto al Containerfile li usiamo cosi' come sono. +COPY vllm-rocm.Containerfile *.tar.gz librccl.so.1.0-* /opt/build-context/ + +# Prende il tag dell'ultima release e lo adatta al target GPU richiesto (es. ...-gfx1151 -> ...-gfx1150). +RUN curl -s https://api.github.com/repos/lemonade-sdk/vllm-rocm/releases/latest -o /tmp/latest.json \ + && LATEST_TAG=$(jq -r '.tag_name' /tmp/latest.json) \ + && if [ -z "$LATEST_TAG" ] || [ "$LATEST_TAG" = "null" ]; then \ + echo "ERRORE: impossibile recuperare il tag latest"; exit 1; \ + fi \ + && echo "Tag latest: $LATEST_TAG" \ + && BASE_TAG=$(echo "$LATEST_TAG" | sed -E 's/-gfx[0-9X]+$//') \ + && TAG="${BASE_TAG}-${GPU_TARGET}" \ + && echo "Tag risolto per target ${GPU_TARGET}: $TAG" \ + && echo "$TAG" > /opt/vllm_rocm_tag.txt \ + && ENCODED_TAG=$(python3 -c "import urllib.parse,sys; print(urllib.parse.quote(sys.argv[1], safe=''))" "$TAG") \ + && echo "Tag URL-encoded: $ENCODED_TAG" \ + && curl -s "https://api.github.com/repos/lemonade-sdk/vllm-rocm/releases/tags/${ENCODED_TAG}" -o /tmp/release.json \ + && echo "--- Asset disponibili in questa release ---" \ + && jq -r '.assets[].name' /tmp/release.json \ + && echo "--- fine lista asset ---" \ + && jq -r '.assets[] | select(.name | test("part[0-9]+-of-[0-9]+\\.tar\\.gz$")) | [.name, .browser_download_url] | @tsv' /tmp/release.json \ + | sort -k1,1 > /tmp/parts.tsv \ + && if [ ! -s /tmp/parts.tsv ]; then \ + echo "ERRORE: release ${TAG} non trovata o senza asset part*-of-*.tar.gz (target ${GPU_TARGET} non ancora pubblicato per questa versione?)"; exit 1; \ + fi \ + && echo "--- Parti trovate, in ordine ---" \ + && cat /tmp/parts.tsv \ + && mkdir -p /opt/vllm-rocm \ + && N=$(wc -l < /tmp/parts.tsv) \ + && i=1 \ + && while [ "$i" -le "$N" ]; do \ + LINE=$(sed -n "${i}p" /tmp/parts.tsv); \ + NAME=$(printf '%s' "$LINE" | cut -f1); \ + URL=$(printf '%s' "$LINE" | cut -f2); \ + if [ -f "/opt/build-context/$NAME" ]; then \ + echo "Trovata localmente ($i/$N): $NAME, copio invece di scaricare"; \ + cp "/opt/build-context/$NAME" "$NAME"; \ + else \ + echo "Scarico ($i/$N): $NAME"; \ + curl -L -o "$NAME" "$URL"; \ + fi; \ + i=$((i + 1)); \ + done \ + && cut -f1 /tmp/parts.tsv | xargs cat | tar xz -C /opt/vllm-rocm \ + && cut -f1 /tmp/parts.tsv | xargs rm -f + +# L'archivio non preserva il bit di esecuzione sui binari/librerie +RUN chmod -R a+rX /opt/vllm-rocm/lib \ + && find /opt/vllm-rocm/bin -type f -exec chmod a+rx {} \; + +# Correggiamo lo shebang degli script in bin/ (es. hipcc, pip), hardcodato con il path della macchina CI originale. +RUN for f in /opt/vllm-rocm/bin/*; do \ + if [ -f "$f" ] && head -c 2 "$f" 2>/dev/null | grep -q '^#!'; then \ + sed -i "1s|^#!.*python3[^ ]*|#!/opt/vllm-rocm/bin/python3|" "$f"; \ + fi; \ + done + +ENV PATH=/opt/vllm-rocm/bin:$PATH +ENV LD_LIBRARY_PATH=/opt/vllm-rocm/lib:$LD_LIBRARY_PATH + +# Symlink stabile lib/python3 verso la cartella versionata reale, la versione di Python cambia tra le release. +RUN PYDIR=$(find /opt/vllm-rocm/lib -maxdepth 1 -name 'python3.*' -type d | head -1) \ + && if [ -z "$PYDIR" ]; then echo "ERRORE: nessuna cartella lib/python3.* trovata"; exit 1; fi \ + && echo "Versione Python rilevata: $(basename "$PYDIR")" \ + && ln -s "$PYDIR" /opt/vllm-rocm/lib/python3 + +# La librccl.so della release e' uno stub senza kernel reali: se librccl.so.1.0- e' presente la usiamo al suo posto, correggendo il RPATH con patchelf verso _rocm_sdk_core/lib. +RUN if [ -f "/opt/build-context/librccl.so.1.0-${GPU_TARGET}" ]; then \ + echo "Sostituisco librccl.so con la build reale per ${GPU_TARGET}"; \ + cp "/opt/build-context/librccl.so.1.0-${GPU_TARGET}" /opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_libraries/lib/librccl.so.1; \ + chmod a+rx /opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_libraries/lib/librccl.so.1; \ + patchelf --set-rpath '$ORIGIN/../../_rocm_sdk_core/lib' /opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_libraries/lib/librccl.so.1; \ + echo "RPATH corretto: $(patchelf --print-rpath /opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_libraries/lib/librccl.so.1)"; \ + else \ + echo "Nessuna librccl.so.1.0-${GPU_TARGET} trovata in /opt/build-context, mantengo lo stub (niente RCCL multi-nodo)"; \ + fi \ + && rm -rf /opt/build-context + +# amdsmi non e' importabile di default in questa build, serve nel PYTHONPATH (vedi lemonade-sdk/vllm-rocm#3). +ENV PYTHONPATH=/opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_core/share/amd_smi + +# La build portable non include pip; ensurepip e' assente, quindi usiamo get-pip.py. +RUN (/opt/vllm-rocm/bin/python3 -m ensurepip --upgrade \ + || (curl -sSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py \ + && /opt/vllm-rocm/bin/python3 /tmp/get-pip.py)) \ + && /opt/vllm-rocm/bin/python3 -m pip install "ray[default]" sentencepiece tiktoken + +# flash_attn qui e' solo metadata senza backend compilato: rotto e mai funzionante, va rimosso per far scattare il fallback nativo di vLLM. +RUN /opt/vllm-rocm/bin/python3 -m pip uninstall -y flash_attn + +# XDG_RUNTIME_DIR scrivibile richiesto dal backend vLLM ROCm in ambienti containerizzati +ENV XDG_RUNTIME_DIR=/tmp/runtime-vllm +RUN mkdir -p /tmp/runtime-vllm && chmod 700 /tmp/runtime-vllm + +WORKDIR /opt/vllm-rocm + +COPY entrypoint.sh /opt/vllm-rocm/entrypoint.sh +RUN chmod +x /opt/vllm-rocm/entrypoint.sh + +ENTRYPOINT ["/opt/vllm-rocm/entrypoint.sh"] diff --git a/containers/vllm/vllm.container b/containers/vllm/vllm.container new file mode 100644 index 0000000..d107acb --- /dev/null +++ b/containers/vllm/vllm.container @@ -0,0 +1,67 @@ +[Container] +ContainerName=vllm +Image=localhost/vllm:rocm +#AutoUpdate=registry + +Network=host + +#Network=internal.network +#PublishPort=6379:6379 + +# Usa Hugging Face Hub per i modelli +Volume=/srv/containers/aitools/models/hf:/root/.cache/huggingface/hub + +# ROCm tuning +AddDevice=/dev/kfd +AddDevice=/dev/dri/renderD128 +PodmanArgs=--group-add=keep-groups --ipc=host --pids-limit=-1 --cap-add=SYS_PTRACE +SecurityLabelType=container_runtime_t + + +# Impostare rete Thunderbolt +Environment=NCCL_SOCKET_IFNAME=eth1 +Environment=GLOO_SOCKET_IFNAME=eth1 + +# Solo per diagnostica dell'errore NCCL attuale, non richiesto dalla guida +# ufficiale AMD per questo hardware (developer.amd.com/playbooks/clustering-rccl) +Environment=NCCL_DEBUG=INFO +# Suggerito direttamente dal warning RCCL: "HIP_FABRIC_API is defined... Rerun +# with RCCL_USE_AMD_SMI_LIB=1 to enable AMD SMI and UALoE fabric support" +Environment=RCCL_USE_AMD_SMI_LIB=1 + + +# Ray / cluster +# CLUSTER=1 -> nodo head (avvia Ray head + vllm serve) +# CLUSTER>1 -> nodo worker (si collega al Ray head e resta attivo) +Environment=CLUSTER=2 +Environment=VLLM_HOST_IP=10.0.0.11 +Environment=RAY_HEAD_ADDRESS=10.0.0.10 +Environment=RAY_PORT=6379 +Environment=NUM_GPUS=1 + +# vLLM serve (usati solo dal nodo head, CLUSTER=1) +Environment=MODEL_PATH=/root/.cache/huggingface/hub/models--unsloth--MiniMax-M2.7-GGUF/snapshots/d2a05ccf69491b03db0cc40b335aec14bdaf7198/UD-IQ4_NL/MiniMax-M2.7-UD-IQ4_NL-00001-of-00004.gguf +#Environment=TOKENIZER=unsloth/MiniMax-M2.7-GGUF +Environment=SERVED_MODEL_NAME=MiniMax-M2.7 +Environment=TRUST_REMOTE_CODE=0 +Environment=SERVE_PORT=7000 +Environment=SERVE_HOST=0.0.0.0 +Environment=MAX_MODEL_LEN=100000 +Environment=GPU_MEMORY_UTILIZATION=0.75 +Environment=TENSOR_PARALLEL_SIZE=2 +Environment=DTYPE=float16 +Environment=KV_CACHE_DTYPE=fp8 + +# Riduce la frammentazione di memoria HIP +Environment=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +# HF +Environment=HF_HOME=/root/.cache/huggingface +Environment=HF_TOKEN= + +[Service] +Restart=on-failure +TimeoutStartSec=15m + +[Install] +WantedBy=multi-user.target default.target \ No newline at end of file