diff --git a/containers/llamacpp/build-mesa.sh b/containers/llamacpp/build-mesa.sh index 02863ab..59f5f4a 100644 --- a/containers/llamacpp/build-mesa.sh +++ b/containers/llamacpp/build-mesa.sh @@ -2,16 +2,11 @@ ############################################################################### # build-mesa.sh # -# Builda Mesa dal repository ufficiale upstream (gitlab.freedesktop.org), -# che su main contiene gia' il fix subgroup per Freedreno (commit -# 185c89084ab7 "freedreno/a6xx: Expose subgroup ops", non ancora presente in -# alcuna release stabile 26.1.x). Produce pacchetti .deb pronti da -# installare, salvati in ./deb/ -# -# Nessun fork necessario: il fix e' gia' su main upstream. +# Builda Mesa dal tag ufficiale mesa-26.2.0-rc3, che include il fix subgroup Freedreno (commit 185c89084ab7) +# Produce pacchetti .deb pronti da installare, salvati in ./deb/ # # Uso: -# ./build-mesa.sh # clona main upstream in /tmp e builda +# ./build-mesa.sh # clona il tag mesa-26.2.0-rc3 in /tmp e builda # ./build-mesa.sh /path/gia/clonato # usa un checkout gia' esistente # ./build-mesa.sh # clona un repo/fork diverso, se serve ############################################################################### @@ -43,6 +38,7 @@ sudo apt-get build-dep -y mesa || { echo UPSTREAM_URL="https://gitlab.freedesktop.org/mesa/mesa.git" +MESA_TAG="mesa-26.2.0-rc3" ARG="${1:-}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -65,18 +61,17 @@ if [ -z "${MESA_SRC:-}" ]; then if [ -d "${CLONE_DIR}/.git" ]; then EXISTING_URL="$(git -C "${CLONE_DIR}" remote get-url origin 2>/dev/null || echo "")" if [ "${EXISTING_URL}" = "${CLONE_URL}" ]; then - echo "=== Checkout gia' presente in ${CLONE_DIR}, aggiorno allineandolo a origin/main ===" - git -C "${CLONE_DIR}" fetch origin main - git -C "${CLONE_DIR}" checkout -B main origin/main - git -C "${CLONE_DIR}" reset --hard origin/main + echo "=== Checkout gia' presente in ${CLONE_DIR}, allineo al tag ${MESA_TAG} ===" + git -C "${CLONE_DIR}" fetch origin "tag" "${MESA_TAG}" + git -C "${CLONE_DIR}" checkout --detach "${MESA_TAG}" else echo "=== ${CLONE_DIR} esiste ma punta a un remote diverso, rifaccio il clone ===" rm -rf "${CLONE_DIR}" - git clone --filter=blob:none --single-branch --branch main "${CLONE_URL}" "${CLONE_DIR}" + git clone --filter=blob:none --branch "${MESA_TAG}" "${CLONE_URL}" "${CLONE_DIR}" fi else - echo "=== Clono ${CLONE_URL} in ${CLONE_DIR} (partial clone, filter=blob:none) ===" - git clone --filter=blob:none --single-branch --branch main "${CLONE_URL}" "${CLONE_DIR}" + echo "=== Clono ${CLONE_URL} in ${CLONE_DIR} (partial clone, filter=blob:none, tag ${MESA_TAG}) ===" + git clone --filter=blob:none --branch "${MESA_TAG}" "${CLONE_URL}" "${CLONE_DIR}" fi MESA_SRC="${CLONE_DIR}" @@ -97,10 +92,10 @@ mkdir -p "${DEB_OUT_DIR}" cd "${MESA_SRC}" SUBGROUP_COMMIT="185c89084ab7a84722b75af6a6026b56945ed44e" if git merge-base --is-ancestor "${SUBGROUP_COMMIT}" HEAD 2>/dev/null; then - echo "OK: commit fix subgroup trovato nella history." + echo "OK: commit fix subgroup trovato nella history di ${MESA_TAG}." else echo "ATTENZIONE: non trovo il commit atteso nella history di questo checkout." >&2 - echo "Verifica di essere sul branch/commit giusto prima di continuare." >&2 + echo "Verifica di essere sul tag/commit giusto prima di continuare." >&2 read -p "Continuare comunque? [y/N] " -n 1 -r echo [[ $REPLY =~ ^[Yy]$ ]] || exit 1 @@ -141,7 +136,7 @@ else PKG_ROOT="${BUILD_DIR}/deb-pkg" PKG_NAME="mesa-freedreno-subgroups" - PKG_VERSION="26.2.0-devel+subgroups1" + PKG_VERSION="26.2.0~rc3+subgroups1" PKG_ARCH="arm64" PKG_DIR="${PKG_ROOT}/${PKG_NAME}_${PKG_VERSION}_${PKG_ARCH}" @@ -157,29 +152,32 @@ Section: libs Priority: optional Architecture: ${PKG_ARCH} Maintainer: local-build -Description: Mesa build from local fork with Freedreno subgroup support - Build locale da fork Mesa (commit 185c89084ab7 "freedreno/a6xx: Expose - subgroup ops", non ancora presente in alcuna release stabile 26.1.x, - disponibile solo su trunk verso 26.2). +Description: Mesa 26.2.0-rc3 build with Freedreno subgroup support + Build locale del tag ufficiale upstream mesa-26.2.0-rc3 (commit 185c89084ab7 + "freedreno/a6xx: Expose subgroup ops", non ancora presente in alcuna + release stabile 26.1.x ne' nei repo delle distro, disponibile a partire + dalla serie 26.2). . Include anche il driver Vulkan Turnip per Freedreno (-Dvulkan-drivers= - freedreno), buildato dallo stesso checkout main upstream cosi' da restare + freedreno), buildato dallo stesso checkout upstream cosi' da restare alla stessa versione/patchset del resto dello stack (GL/EGL/GBM/rusticl). . Sostituisce i file installati da mesa-libgallium / mesa-opencl-icd / - mesa-vulkan-drivers di Debian con la build patchata. Da usare solo per test. + mesa-vulkan-drivers della distro con la build 26.2.0-rc3. EOF cat > "${PKG_DIR}/DEBIAN/README-patch.md" << 'EOF' -# Nota sulla patch inclusa +# Nota sul fix incluso Commit: 185c89084ab7 "freedreno/a6xx: Expose subgroup ops" Autore: upstream Mesa (author date 2026-02-11, committer/merge date 2026-05-18) -Motivo assenza in Debian: il branch stabile 26.1 e' stato diramato da main il -2026-04-14 (tag mesa-26.1.0 il 2026-05-06), quindi questo fix e' arrivato su -main DOPO la diramazione. Essendo una feature (non un bugfix), non verra' -backportato al branch 26.1.x per policy standard di Mesa. Il fix esiste solo -su trunk, verso la futura 26.2. +Motivo assenza nei repo delle distro: il branch stabile 26.1 e' stato +diramato da main il 2026-04-14 (tag mesa-26.1.0 il 2026-05-06), quindi +questo fix e' arrivato su main DOPO la diramazione. Essendo una feature (non +un bugfix), non e' stato backportato al branch 26.1.x per policy standard di +Mesa. Il fix e' incluso a partire dal tag ufficiale mesa-26.2.0-rc3, ancora +troppo recente per essere presente nei repository delle distro (es. Ubuntu +26.04 pacchetta Mesa 26.0.8). Verificato (vedi indagine precedente): - Nessuna dipendenza da ioctl/kernel driver msm: i campi @@ -190,12 +188,12 @@ Verificato (vedi indagine precedente): questo fix e HEAD: e' completo e autosufficiente, non un WIP parziale. Questo pacchetto installa i file prodotti da una build meson/ninja pulita -di questo checkout, con gallium-drivers=freedreno, vulkan-drivers=freedreno -(Turnip) e rusticl abilitato per freedreno. +del tag mesa-26.2.0-rc3, con gallium-drivers=freedreno, vulkan-drivers= +freedreno (Turnip) e rusticl abilitato per freedreno. Nota: il pacchetto include il driver ICD Vulkan (libvulkan_freedreno.so + freedreno_icd.json) ma NON il Vulkan loader (libvulkan.so.1), che resta -il pacchetto standard Debian "libvulkan1" installato separatamente nel +il pacchetto standard "libvulkan1" installato separatamente nel Containerfile. EOF diff --git a/containers/llamacpp/llamacpp-adreno.Containerfile b/containers/llamacpp/llamacpp-adreno.Containerfile index e94f32d..be68d15 100644 --- a/containers/llamacpp/llamacpp-adreno.Containerfile +++ b/containers/llamacpp/llamacpp-adreno.Containerfile @@ -5,7 +5,7 @@ ### BUILD: podman build -t llamacpp:adreno-arm64 -f llamacpp-adreno.Containerfile . ### Export: podman save -o /home/badstorm/llamacpp-adreno-arm64.tar localhost/llamacpp:adreno-arm64 -FROM debian:13-slim +FROM ubuntu:26.04 USER root EXPOSE 8090 @@ -25,7 +25,7 @@ RUN apt-get update \ WORKDIR /app -# Install Mesa Freedreno DEB with Adreno Vulkan support and subgroup operations +# Mesa 26.2.0-rc3 buildato da noi (build-mesa.sh): su Debian experimental e' solo amd64/i386, niente arm64 ancora # Remove conflicting Mesa packages first to avoid file conflicts COPY deb/mesa-freedreno-subgroups_*.deb /tmp/ RUN apt-get update \ diff --git a/containers/llamacpp/llamacpp-mistral.Containerfile b/containers/llamacpp/llamacpp-mistral.Containerfile deleted file mode 100644 index 56a8906..0000000 --- a/containers/llamacpp/llamacpp-mistral.Containerfile +++ /dev/null @@ -1,51 +0,0 @@ -### LLaMACpp Builder Container with Vulkan for GPUs -### Multi-stage: download stage with pre-built binaries, runtime stage with only runtime libraries -### -### BUILD: podman build -t llamacpp:vulkan-amd64 -f llamacpp.Containerfile . -### Export: podman save -o /home/badstorm/llamacpp-vulkan-amd64.tar localhost/llamacpp:vulkan-amd64 - - -FROM ubuntu:24.04 - -USER root -EXPOSE 8090 - -RUN apt-get update \ - && apt-get install -y curl tar grep sed git ffmpeg nano python3-pip python3 python3-wheel \ - && pip install --break-system-packages --upgrade setuptools \ - && pip install --break-system-packages -U "huggingface_hub[cli]" \ - && if [ -f requirements.txt ]; then pip install --break-system-packages -r requirements.txt; fi \ - && apt autoremove -y \ - && apt clean -y \ - && rm -rf /tmp/* /var/tmp/* \ - && rm -rf /var/lib/apt/lists/* \ - && find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \ - && find /var/cache -type f -delete - -WORKDIR /app - -RUN VERSION=$(curl -s https://api.github.com/repos/ggml-org/llama.cpp/releases/latest | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": "\([^"]*\)".*/\1/') \ - && echo "Last llama.cpp version: $VERSION" \ - && curl -L https://github.com/ggml-org/llama.cpp/releases/download/${VERSION}/llama-${VERSION}-bin-ubuntu-vulkan-x64.tar.gz -o llama.tar.gz \ - && tar -xzf llama.tar.gz -C . --strip-components=1 \ - && rm llama.tar.gz - -RUN chmod +x /app/llama-server - -WORKDIR /app - -ENV PATH=/app:$PATH -ENV LD_LIBRARY_PATH=/app:$LD_LIBRARY_PATH -ENV HF_HUB_ENABLE_HF_TRANSFER=1 -#ENV HF_HOME= -#ENV HUGGING_FACE_HUB_TOKEN= -ENV LLAMA_ARG_HOST=0.0.0.0 -ENV LLAMA_ARG_PORT=8090 -ENV LLAMA_ARG_HF_REPO=unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF:Q2_K -#ENV LLAMA_ARG_MMPROJ_URL= -ENV LLAMA_ARG_NO_MMAP=true -ENV LLAMA_ARG_CTX_SIZE=128000 -#ENV LLAMA_API_KEY="" - -ENTRYPOINT ["/app/llama-server"] -CMD ["--no-warmup"] \ No newline at end of file diff --git a/containers/llamacpp/llamacpp-opencl.Containerfile b/containers/llamacpp/llamacpp-opencl.Containerfile index 6cf9baf..ef77c4f 100644 --- a/containers/llamacpp/llamacpp-opencl.Containerfile +++ b/containers/llamacpp/llamacpp-opencl.Containerfile @@ -8,12 +8,11 @@ ### backend OpenCL di llama.cpp (nessun kernel mul_mv_iq2_*/iq3_* esiste in ### ggml/src/ggml-opencl/kernels/), indipendentemente dalle patch qui sotto. ### Q2_K/Q3_K invece SONO supportati (riusano i kernel Q4_K). -FROM debian:13-slim +FROM ubuntu:26.04 USER root EXPOSE 8090 -# Mesa patchata (fix subgroup Freedreno a6xx, commit 185c89084ab7, non in -# alcuna release Debian/backports). Generata da build-mesa.sh in ./deb. +# Mesa 26.2.0-rc3 buildata da noi (build-mesa.sh in ./deb): su Debian experimental e' solo amd64/i386, niente arm64 ancora COPY deb/*.deb /tmp/mesa-deb/ RUN apt-get update \ diff --git a/containers/llamacpp/llamacpp-rocm.Containerfile b/containers/llamacpp/llamacpp-rocm.Containerfile new file mode 100644 index 0000000..d5180f3 --- /dev/null +++ b/containers/llamacpp/llamacpp-rocm.Containerfile @@ -0,0 +1,67 @@ +### LLaMACpp Builder Container with rocm for GPUs +### Usa le build lemonade-sdk/llamacpp-rocm (per-target: gfx90a, gfx908, gfx103X, gfx110X, gfx1150, gfx1151, gfx120X) +### +### BUILD (gfx1151, default): +### podman build -t llamacpp:rocm -f llamacpp-rocm.Containerfile . +### BUILD (altro target): +### podman build --build-arg GPU_TARGET=gfx110X -t llamacpp:rocm -f llamacpp-rocm.Containerfile . +### Export: podman save -o /home/badstorm/llamacpp-rocm.tar localhost/llamacpp:rocm + +FROM ubuntu:26.04 + +ARG GPU_TARGET=gfx1151 + +USER root +EXPOSE 8090 + +RUN apt-get update \ + && apt-get install -y curl unzip jq tar grep sed git ffmpeg nano python3-pip python3 python3-wheel \ + && pip install --break-system-packages --upgrade setuptools \ + && pip install --break-system-packages -U "huggingface_hub[cli]" \ + && if [ -f requirements.txt ]; then pip install --break-system-packages -r requirements.txt; fi \ + && apt autoremove -y \ + && apt clean -y \ + && rm -rf /tmp/* /var/tmp/* \ + && rm -rf /var/lib/apt/lists/* \ + && find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \ + && find /var/cache -type f -delete + +WORKDIR /app + +# Prende l'ultima release lemonade-sdk/llamacpp-rocm per il target GPU richiesto +RUN curl -s https://api.github.com/repos/lemonade-sdk/llamacpp-rocm/releases/latest -o /tmp/latest.json \ + && TAG=$(jq -r '.tag_name' /tmp/latest.json) \ + && if [ -z "$TAG" ] || [ "$TAG" = "null" ]; then \ + echo "ERRORE: impossibile recuperare il tag latest"; exit 1; \ + fi \ + && echo "Ultima versione llamacpp-rocm: $TAG" \ + && ASSET_URL=$(jq -r --arg t "$GPU_TARGET" '.assets[] | select(.name | test("^llama-.*-ubuntu-rocm-" + $t + "-x64\\.zip$")) | .browser_download_url' /tmp/latest.json) \ + && if [ -z "$ASSET_URL" ]; then \ + echo "ERRORE: nessun asset per target ${GPU_TARGET} nella release ${TAG}"; \ + echo "Asset disponibili:"; jq -r '.assets[].name' /tmp/latest.json; \ + exit 1; \ + fi \ + && echo "Scarico: $ASSET_URL" \ + && curl -L "$ASSET_URL" -o llama.zip \ + && unzip -q llama.zip -d /app \ + && rm llama.zip /tmp/latest.json + +RUN chmod +x /app/llama-server + +WORKDIR /app + +ENV PATH=/app:$PATH +ENV LD_LIBRARY_PATH=/app:$LD_LIBRARY_PATH +ENV HF_HUB_ENABLE_HF_TRANSFER=1 +#ENV HF_HOME= +#ENV HUGGING_FACE_HUB_TOKEN= +ENV LLAMA_ARG_HOST=0.0.0.0 +ENV LLAMA_ARG_PORT=8090 +ENV LLAMA_ARG_HF_REPO=unsloth/Qwen3.5-35B-A3B-GGUF:Q2_K_XL +#ENV LLAMA_ARG_MMPROJ_URL= +ENV LLAMA_ARG_NO_MMAP=true +ENV LLAMA_ARG_CTX_SIZE=128000 +#ENV LLAMA_API_KEY="" + +ENTRYPOINT ["/app/llama-server"] +CMD ["--no-warmup"] diff --git a/containers/llamacpp/llamacpp-rpc.container b/containers/llamacpp/llamacpp-rpc.container new file mode 100644 index 0000000..923c92a --- /dev/null +++ b/containers/llamacpp/llamacpp-rpc.container @@ -0,0 +1,26 @@ +[Container] +ContainerName=llamacpp-rpc +Image=localhost/llamacpp:vulkan-amd64 +#AutoUpdate=registry +Network=internal.network +# NON pubblicare sull'host: backend RPC insicuro, solo raggiungibile dagli altri container su internal.network +#PublishPort=50052:50052 + +# ROCm tuning +AddDevice=/dev/dri/renderD128 +PodmanArgs=--group-add=keep-groups --ipc=host +SecurityLabelType=container_runtime_t + +# Cache locale dei tensori (-c), evita di ritrasferirli in rete a ogni load +Volume=/srv/containers/aitools/models/rpc-cache:/root/.cache/llama.cpp/rpc + +# Worker RPC: espone le device locali ai server llama.cpp remoti, non e' il server principale +Entrypoint=/app/ggml-rpc-server +Exec=-H 0.0.0.0 -p 50052 -c + +[Service] +Restart=on-failure +TimeoutStartSec=15m + +[Install] +WantedBy=multi-user.target default.target diff --git a/containers/llamacpp/llamacpp-vulkan.Containerfile b/containers/llamacpp/llamacpp-vulkan.Containerfile index ba0c731..120c233 100644 --- a/containers/llamacpp/llamacpp-vulkan.Containerfile +++ b/containers/llamacpp/llamacpp-vulkan.Containerfile @@ -1,11 +1,11 @@ ### LLaMACpp Builder Container with Vulkan for GPUs ### Multi-stage: download stage with pre-built binaries, runtime stage with only runtime libraries ### -### BUILD: podman build -t llamacpp:vulkan-amd64 -f llamacpp-vulkan.Containerfile . -### Export: podman save -o /home/badstorm/llamacpp-vulkan-amd64.tar localhost/llamacpp:vulkan-amd64 +### BUILD: podman build -t llamacpp:vulkan -f llamacpp-vulkan.Containerfile . +### Export: podman save -o /home/badstorm/llamacpp-vulkan.tar localhost/llamacpp:vulkan -FROM debian:13-slim +FROM ubuntu:26.04 USER root EXPOSE 8090 diff --git a/containers/llamacpp/llamacpp.container b/containers/llamacpp/llamacpp.container index 32846e7..c3c16bf 100644 --- a/containers/llamacpp/llamacpp.container +++ b/containers/llamacpp/llamacpp.container @@ -1,7 +1,11 @@ [Container] ContainerName=llamacpp -Image=localhost/llamacpp:vulkan-amd64 -#AutoUpdate=registry + +# Vlukan +Image=localhost/llamacpp:vulkan +# ROCm +#Image=localhost/llamacpp:rocm + Network=internal.network PublishPort=8090:8090 @@ -25,6 +29,9 @@ Environment=LLAMA_ARG_HF_REPO=unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF:Q2_K Environment=LLAMA_ARG_SPEC_TYPE=draft-mtp Environment=LLAMA_ARG_SPEC_DRAFT_N_MAX=6 +# RPC — worker ggml-rpc-server (llamacpp-rpc.container), lista host:port; backend insicuro, solo su internal.network +#Environment=LLAMA_ARG_RPC=llamacpp-rpc:50052 + # HF Environment=HF_HOME=/root/.cache/huggingface Environment=HF_TOKEN=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 diff --git a/containers/vllm/build-librccl.sh b/containers/vllm/build-librccl.sh new file mode 100755 index 0000000..206f8fc --- /dev/null +++ b/containers/vllm/build-librccl.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Risolve/scarica (con cache locale) la tarball ROCm/TheRock, builda rocm-librccl.Containerfile +# ed estrae automaticamente librccl.so.1.0 nella stessa directory dello script. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +GPU_TARGETS="${GPU_TARGETS:-gfx1150;gfx1151}" +ROCM_AMDGPU_FAMILY="${ROCM_AMDGPU_FAMILY:-gfx110X-all}" +ROCM_RELEASE_TYPE="${ROCM_RELEASE_TYPE:-nightlies}" +ROCM_VERSION="${ROCM_VERSION:-}" +RCCL_REPO="${RCCL_REPO:-https://github.com/ROCm/rocm-systems.git}" +RCCL_BRANCH="${RCCL_BRANCH:-develop}" +IMAGE_TAG="${IMAGE_TAG:-vllm:amd-rccl}" + +case "$ROCM_RELEASE_TYPE" in + stable) LISTING_URL="https://repo.amd.com/rocm/tarball/" ;; + prereleases) LISTING_URL="https://rocm.prereleases.amd.com/tarball/" ;; + nightlies) LISTING_URL="https://rocm.nightlies.amd.com/tarball/" ;; + *) echo "ERRORE: ROCM_RELEASE_TYPE sconosciuto: $ROCM_RELEASE_TYPE" >&2; exit 1 ;; +esac + +PREFIX="therock-dist-linux-${ROCM_AMDGPU_FAMILY}-" + +if [ -z "$ROCM_VERSION" ]; then + echo "Risolvo l'ultima versione ROCm disponibile (${ROCM_RELEASE_TYPE}, ${ROCM_AMDGPU_FAMILY})..." + ROCM_VERSION=$(curl -fsSL "$LISTING_URL" \ + | grep -oE "\"name\": \"${PREFIX}[^\"]+\.tar\.gz\"" \ + | sed -E "s/^\"name\": \"${PREFIX}//; s/\.tar\.gz\"\$//" \ + | sort -V | tail -1) + if [ -z "$ROCM_VERSION" ]; then + echo "ERRORE: nessuna tarball trovata per ${ROCM_AMDGPU_FAMILY} (${ROCM_RELEASE_TYPE}) su ${LISTING_URL}" >&2 + exit 1 + fi + echo "Versione risolta: $ROCM_VERSION" +fi + +TARBALL_NAME="therock-dist-linux-${ROCM_AMDGPU_FAMILY}-${ROCM_VERSION}.tar.gz" +TARBALL_URL="${LISTING_URL}${TARBALL_NAME}" + +# Rimuove eventuali tarball ROCm precedenti (altra versione) per non confondere il COPY per-glob del Containerfile. +find . -maxdepth 1 -name 'therock-dist-linux-*.tar.gz' ! -name "$TARBALL_NAME" -exec rm -f {} \; + +if [ -f "$TARBALL_NAME" ]; then + echo "Tarball gia' presente localmente: $TARBALL_NAME, non la riscarico" +else + echo "Scarico $TARBALL_NAME da $TARBALL_URL" + curl -fsSL -o "${TARBALL_NAME}.part" "$TARBALL_URL" + mv "${TARBALL_NAME}.part" "$TARBALL_NAME" +fi + +echo "Build immagine podman ($IMAGE_TAG)..." +podman build --pull=never \ + --build-arg ROCM_VERSION="$ROCM_VERSION" \ + --build-arg ROCM_AMDGPU_FAMILY="$ROCM_AMDGPU_FAMILY" \ + --build-arg ROCM_RELEASE_TYPE="$ROCM_RELEASE_TYPE" \ + --build-arg GPU_TARGETS="$GPU_TARGETS" \ + --build-arg RCCL_REPO="$RCCL_REPO" \ + --build-arg RCCL_BRANCH="$RCCL_BRANCH" \ + -t "$IMAGE_TAG" \ + -f rocm-librccl.Containerfile . + +echo "Estrazione librccl.so.1.0..." +id=$(podman create "$IMAGE_TAG") +podman cp "$id:/output/lib/librccl.so.1" ./librccl.so.1.0 +podman rm "$id" >/dev/null + +echo "Fatto: librccl.so.1.0 salvato in ${SCRIPT_DIR}/librccl.so.1.0" diff --git a/containers/vllm/entrypoint.sh b/containers/vllm/entrypoint.sh index b0c1617..129b98c 100755 --- a/containers/vllm/entrypoint.sh +++ b/containers/vllm/entrypoint.sh @@ -1,15 +1,52 @@ #!/usr/bin/env bash set -euo pipefail +# CLUSTER=0 -> singolo nodo: avvia vllm serve senza Ray (nessun cluster) # CLUSTER=1 -> nodo head: avvia Ray head e poi vllm serve # CLUSTER>1 -> nodo worker: si collega al Ray head e resta attivo CLUSTER="${CLUSTER:-1}" -: "${VLLM_HOST_IP:?VLLM_HOST_IP non impostato}" RAY_PORT="${RAY_PORT:-6379}" NUM_GPUS="${NUM_GPUS:-1}" -if [ "$CLUSTER" -eq 1 ]; then +if [ "$CLUSTER" -eq 0 ]; then + : "${MODEL_PATH:?MODEL_PATH non impostato}" + + # TOKENIZER e TRUST_REMOTE_CODE sono opzionali: se non impostate, vllm + # usa il tokenizer embedded nel GGUF e non esegue codice remoto. + EXTRA_ARGS=() + if [ -n "${TOKENIZER:-}" ]; then + EXTRA_ARGS+=(--tokenizer "${TOKENIZER}") + fi + if [ "${TRUST_REMOTE_CODE:-0}" = "1" ]; then + EXTRA_ARGS+=(--trust-remote-code) + fi + if [ -n "${SERVED_MODEL_NAME:-}" ]; then + EXTRA_ARGS+=(--served-model-name "${SERVED_MODEL_NAME}") + fi + if [ "${ENABLE_AUTO_TOOL_CHOICE:-0}" = "1" ]; then + EXTRA_ARGS+=(--enable-auto-tool-choice) + : "${TOOL_CALL_PARSER:?TOOL_CALL_PARSER richiesto se ENABLE_AUTO_TOOL_CHOICE=1}" + EXTRA_ARGS+=(--tool-call-parser "${TOOL_CALL_PARSER}") + fi + if [ -n "${REASONING_PARSER:-}" ]; then + EXTRA_ARGS+=(--reasoning-parser "${REASONING_PARSER}") + fi + + echo "[entrypoint] Modalita' singolo nodo (no cluster): avvio vllm serve senza Ray" + exec vllm serve "${MODEL_PATH}" \ + "${EXTRA_ARGS[@]}" \ + --port "${SERVE_PORT:-7000}" \ + --host "${SERVE_HOST:-0.0.0.0}" \ + --max-model-len "${MAX_MODEL_LEN:-32768}" \ + --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.9}" \ + --tensor-parallel-size 1 \ + --enforce-eager \ + --dtype "${DTYPE:-float16}" \ + --kv-cache-dtype "${KV_CACHE_DTYPE:-fp8}" + +elif [ "$CLUSTER" -eq 1 ]; then + : "${VLLM_HOST_IP:?VLLM_HOST_IP non impostato}" echo "[entrypoint] Nodo head: avvio Ray head su ${VLLM_HOST_IP}:${RAY_PORT}" ray start --head --port="${RAY_PORT}" --node-ip-address="${VLLM_HOST_IP}" --num-gpus="${NUM_GPUS}" \ --dashboard-host=0.0.0.0 @@ -28,6 +65,14 @@ if [ "$CLUSTER" -eq 1 ]; then if [ -n "${SERVED_MODEL_NAME:-}" ]; then EXTRA_ARGS+=(--served-model-name "${SERVED_MODEL_NAME}") fi + if [ "${ENABLE_AUTO_TOOL_CHOICE:-0}" = "1" ]; then + EXTRA_ARGS+=(--enable-auto-tool-choice) + : "${TOOL_CALL_PARSER:?TOOL_CALL_PARSER richiesto se ENABLE_AUTO_TOOL_CHOICE=1}" + EXTRA_ARGS+=(--tool-call-parser "${TOOL_CALL_PARSER}") + fi + if [ -n "${REASONING_PARSER:-}" ]; then + EXTRA_ARGS+=(--reasoning-parser "${REASONING_PARSER}") + fi echo "[entrypoint] Avvio vllm serve" exec vllm serve "${MODEL_PATH}" \ @@ -38,10 +83,13 @@ if [ "$CLUSTER" -eq 1 ]; then --gpu-memory-utilization "${GPU_MEMORY_UTILIZATION:-0.9}" \ --tensor-parallel-size "${TENSOR_PARALLEL_SIZE:-2}" \ --distributed-executor-backend ray \ + --distributed-timeout-seconds "${DISTRIBUTED_TIMEOUT_SECONDS:-600}" \ --enforce-eager \ --dtype "${DTYPE:-float16}" \ --kv-cache-dtype "${KV_CACHE_DTYPE:-fp8}" + else + : "${VLLM_HOST_IP:?VLLM_HOST_IP non impostato}" : "${RAY_HEAD_ADDRESS:?RAY_HEAD_ADDRESS non impostato (IP del nodo head)}" echo "[entrypoint] Nodo worker: mi collego al Ray head su ${RAY_HEAD_ADDRESS}:${RAY_PORT}" ray start --address="${RAY_HEAD_ADDRESS}:${RAY_PORT}" --node-ip-address="${VLLM_HOST_IP}" --num-gpus="${NUM_GPUS}" diff --git a/containers/vllm/rocm-librccl.Containerfile b/containers/vllm/rocm-librccl.Containerfile new file mode 100644 index 0000000..9a595a1 --- /dev/null +++ b/containers/vllm/rocm-librccl.Containerfile @@ -0,0 +1,61 @@ +### Container di BUILD (non di runtime) per compilare RCCL dal monorepo UFFICIALE ROCm/rocm-systems (develop, projects/rccl) con lo script install.sh ufficiale, usando il toolchain ufficiale TheRock (piu' aggiornato della vecchia immagine AMD Ryzen AI). +### Base: immagine build manylinux di TheRock (AlmaLinux/RPM, non Ubuntu), con gcc-toolset-13/cmake/ninja/patchelf gia' pronti; il tarball ROCm ci aggiunge hipcc. +### TheRock supporta ufficialmente sia gfx1150 che gfx1151 (RDNA3.5, vedi SUPPORTED_GPUS.md): si puo' quindi buildare UN SOLO librccl.so con kernel per entrambe le architetture insieme. +### +### Uso raccomandato: ./build-librccl.sh (risolve/scarica/riusa la tarball ROCm, builda, estrae librccl.so.1.0 automaticamente). +### +### Uso manuale: +### podman build --pull=never --build-arg ROCM_VERSION= -t vllm:amd-rccl -f rocm-librccl.Containerfile . +### id=$(podman create vllm:amd-rccl) +### podman cp $id:/output/lib/librccl.so.1 ./librccl.so.1.0 +### podman rm $id + +FROM ghcr.io/rocm/therock_build_manylinux_x86_64:latest + +ARG ROCM_VERSION= +ARG ROCM_AMDGPU_FAMILY=gfx110X-all +ARG ROCM_RELEASE_TYPE=nightlies +ARG GPU_TARGETS=gfx1150;gfx1151 +ARG RCCL_REPO=https://github.com/ROCm/rocm-systems.git +ARG RCCL_BRANCH=develop + +# Pacchetti RPM mancanti nella base manylinux, necessari per RCCL (equivalenti di libdrm-dev/libnuma-dev/pkg-config su Ubuntu). +RUN dnf install -y libdrm-devel numactl-devel pkgconfig \ + && dnf clean all + +WORKDIR /build + +# Se una tarball ROCm e' gia' presente accanto al Containerfile (scaricata da build-librccl.sh) la usiamo cosi' com'e', senza rifare il download in rete. +COPY rocm-librccl.Containerfile therock-dist-linux-*.tar.gz /opt/build-context/ + +RUN TARBALL=$(find /opt/build-context -maxdepth 1 -name 'therock-dist-linux-*.tar.gz' | head -1); \ + if [ -n "$TARBALL" ]; then \ + echo "Uso tarball ROCm locale: $TARBALL"; \ + mkdir -p /opt/rocm-local; \ + tar -xzf "$TARBALL" -C /opt/rocm-local; \ + ln -sfn /opt/rocm-local /opt/rocm; \ + else \ + echo "Nessuna tarball locale trovata, scarico ed installo con lo script ufficiale TheRock"; \ + if [ -z "${ROCM_VERSION}" ]; then echo "ERRORE: ROCM_VERSION richiesto quando non c'e' una tarball locale in /opt/build-context"; exit 1; fi; \ + curl -fsSL -o /tmp/install_rocm_tarball.sh https://raw.githubusercontent.com/ROCm/TheRock/main/dockerfiles/install_rocm_tarball.sh; \ + chmod +x /tmp/install_rocm_tarball.sh; \ + /tmp/install_rocm_tarball.sh "${ROCM_VERSION}" "${ROCM_AMDGPU_FAMILY}" "${ROCM_RELEASE_TYPE}"; \ + rm -f /tmp/install_rocm_tarball.sh; \ + fi \ + && rm -rf /opt/build-context + +ENV ROCM_PATH=/opt/rocm +ENV PATH=/opt/rocm/bin:$PATH + +# Sparse checkout: il monorepo e' enorme, prendiamo solo projects/rccl. +RUN git clone --depth 1 --branch "${RCCL_BRANCH}" --filter=blob:none --sparse "${RCCL_REPO}" rocm-systems \ + && cd rocm-systems \ + && git sparse-checkout set projects/rccl \ + && git submodule update --init --recursive --depth=1 -- projects/rccl || true + +# Script di build/install ufficiale del progetto; CMake vuole liste separate da ";" (non ",") per passare piu' target GPU in un solo binario. +RUN cd /build/rocm-systems/projects/rccl \ + && ./install.sh -i --amdgpu_targets="${GPU_TARGETS}" --prefix=/output -j "$(nproc)" + +RUN echo "--- Output ---" \ + && find /output -iname 'librccl.so*' -exec ls -la {} \; diff --git a/containers/vllm/vllm-amd.Containerfile b/containers/vllm/vllm-amd.Containerfile deleted file mode 100644 index d4cb91c..0000000 --- a/containers/vllm/vllm-amd.Containerfile +++ /dev/null @@ -1,50 +0,0 @@ -### Container di BUILD (non di runtime) per compilare RCCL dal monorepo UFFICIALE ROCm/rocm-systems (develop, projects/rccl) con lo script install.sh ufficiale, usando il toolchain ufficiale TheRock (piu' aggiornato della vecchia immagine AMD Ryzen AI). -### Base: immagine build manylinux di TheRock (AlmaLinux/RPM, non Ubuntu), con gcc-toolset-13/cmake/ninja/patchelf gia' pronti; il tarball ROCm ci aggiunge hipcc. -### TheRock supporta ufficialmente sia gfx1150 che gfx1151 (RDNA3.5, vedi SUPPORTED_GPUS.md) dalla versione 10.1.0: si puo' quindi buildare UN SOLO librccl.so con kernel per entrambe le architetture insieme. -### -### BUILD (entrambi i target in un solo file, valido per head e worker): -### podman build --pull=never -t vllm:amd-rccl -f vllm-amd.Containerfile . -### BUILD per un solo target (es. se serve isolare un problema specifico): -### podman build --pull=never --build-arg GPU_TARGETS=gfx1150 -t vllm:amd-rccl-gfx1150 -f vllm-amd.Containerfile . -### -### ESTRARRE IL FILE COMPILATO: -### id=$(podman create vllm:amd-rccl) -### podman cp $id:/output/lib/librccl.so.1 ./librccl.so.1.0 -### podman rm $id - -FROM ghcr.io/rocm/therock_build_manylinux_x86_64:latest - -ARG ROCM_VERSION=10.1.0 -ARG ROCM_AMDGPU_FAMILY=gfx110X-all -ARG ROCM_RELEASE_TYPE=stable -ARG GPU_TARGETS=gfx1150,gfx1151 -ARG RCCL_REPO=https://github.com/ROCm/rocm-systems.git -ARG RCCL_BRANCH=develop - -# Pacchetti RPM mancanti nella base manylinux, necessari per RCCL (equivalenti di libdrm-dev/libnuma-dev/pkg-config su Ubuntu). -RUN dnf install -y libdrm-devel numactl-devel pkgconfig \ - && dnf clean all - -WORKDIR /build - -# Script ufficiale TheRock per installare un tarball ROCm gia' pronto (include hipcc) per la famiglia GPU richiesta. -RUN curl -fsSL -o install_rocm_tarball.sh https://raw.githubusercontent.com/ROCm/TheRock/main/dockerfiles/install_rocm_tarball.sh \ - && chmod +x install_rocm_tarball.sh \ - && ./install_rocm_tarball.sh "${ROCM_VERSION}" "${ROCM_AMDGPU_FAMILY}" "${ROCM_RELEASE_TYPE}" \ - && rm -f install_rocm_tarball.sh - -ENV ROCM_PATH=/opt/rocm -ENV PATH=/opt/rocm/bin:$PATH - -# Sparse checkout: il monorepo e' enorme, prendiamo solo projects/rccl. -RUN git clone --depth 1 --branch "${RCCL_BRANCH}" --filter=blob:none --sparse "${RCCL_REPO}" rocm-systems \ - && cd rocm-systems \ - && git sparse-checkout set projects/rccl \ - && git submodule update --init --recursive --depth=1 -- projects/rccl || true - -# Script di build/install ufficiale del progetto; con la virgola si possono passare piu' target GPU in un solo binario. -RUN cd /build/rocm-systems/projects/rccl \ - && ./install.sh -i --amdgpu_targets="${GPU_TARGETS}" --prefix=/output -j "$(nproc)" - -RUN echo "--- Output ---" \ - && find /output -iname 'librccl.so*' -exec ls -la {} \; diff --git a/containers/vllm/vllm-rocm.Containerfile b/containers/vllm/vllm-rocm.Containerfile index 4fe774c..3b93299 100644 --- a/containers/vllm/vllm-rocm.Containerfile +++ b/containers/vllm/vllm-rocm.Containerfile @@ -16,8 +16,8 @@ RUN apt-get update \ WORKDIR /opt -# Se i .tar.gz della release e/o librccl.so.1.0- sono gia' presenti accanto al Containerfile li usiamo cosi' come sono. -COPY vllm-rocm.Containerfile *.tar.gz librccl.so.1.0-* /opt/build-context/ +# Se i .tar.gz della release e/o librccl.so.1.0 (build TheRock, unica per tutti i target) sono gia' presenti accanto al Containerfile li usiamo cosi' come sono. +COPY vllm-rocm.Containerfile *.tar.gz librccl.so.1.0* /opt/build-context/ # Prende il tag dell'ultima release e lo adatta al target GPU richiesto (es. ...-gfx1151 -> ...-gfx1150). RUN curl -s https://api.github.com/repos/lemonade-sdk/vllm-rocm/releases/latest -o /tmp/latest.json \ @@ -82,15 +82,21 @@ RUN PYDIR=$(find /opt/vllm-rocm/lib -maxdepth 1 -name 'python3.*' -type d | head && echo "Versione Python rilevata: $(basename "$PYDIR")" \ && ln -s "$PYDIR" /opt/vllm-rocm/lib/python3 -# La librccl.so della release e' uno stub senza kernel reali: se librccl.so.1.0- e' presente la usiamo al suo posto, correggendo il RPATH con patchelf verso _rocm_sdk_core/lib. -RUN if [ -f "/opt/build-context/librccl.so.1.0-${GPU_TARGET}" ]; then \ - echo "Sostituisco librccl.so con la build reale per ${GPU_TARGET}"; \ - cp "/opt/build-context/librccl.so.1.0-${GPU_TARGET}" /opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_libraries/lib/librccl.so.1; \ +# La librccl.so della release e' uno stub senza kernel reali: se e' presente una build reale (TheRock, unica per tutti i target: librccl.so.1.0; oppure, per compatibilita' con vecchie build per-target: librccl.so.1.0-) la usiamo al suo posto, correggendo il RPATH con patchelf verso _rocm_sdk_core/lib. +RUN RCCL_SRC=""; \ + if [ -f "/opt/build-context/librccl.so.1.0" ]; then \ + RCCL_SRC="/opt/build-context/librccl.so.1.0"; \ + elif [ -f "/opt/build-context/librccl.so.1.0-${GPU_TARGET}" ]; then \ + RCCL_SRC="/opt/build-context/librccl.so.1.0-${GPU_TARGET}"; \ + fi; \ + if [ -n "$RCCL_SRC" ]; then \ + echo "Sostituisco librccl.so con la build reale: $RCCL_SRC"; \ + cp "$RCCL_SRC" /opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_libraries/lib/librccl.so.1; \ chmod a+rx /opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_libraries/lib/librccl.so.1; \ patchelf --set-rpath '$ORIGIN/../../_rocm_sdk_core/lib' /opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_libraries/lib/librccl.so.1; \ echo "RPATH corretto: $(patchelf --print-rpath /opt/vllm-rocm/lib/python3/site-packages/_rocm_sdk_libraries/lib/librccl.so.1)"; \ else \ - echo "Nessuna librccl.so.1.0-${GPU_TARGET} trovata in /opt/build-context, mantengo lo stub (niente RCCL multi-nodo)"; \ + echo "Nessuna librccl.so.1.0 trovata in /opt/build-context, mantengo lo stub (niente RCCL multi-nodo)"; \ fi \ && rm -rf /opt/build-context diff --git a/containers/vllm/vllm.caddy b/containers/vllm/vllm.caddy new file mode 100644 index 0000000..3a7f8f6 --- /dev/null +++ b/containers/vllm/vllm.caddy @@ -0,0 +1,26 @@ +[domain] { + import gateway_error + + # Security Headers + header { + Strict-Transport-Security "max-age=31536000; includeSubDomains" + X-Frame-Options SAMEORIGIN + X-Content-Type-Options nosniff + X-Xss-Protection "1; mode=block" + } + + request_body { + max_size 512MB + } + + # Blocca accesso a file nascosti + @hidden path_regexp ^/\.ht + respond @hidden 404 + + # API OpenAI-compatible vLLM (porta SERVE_PORT, nodo head del cluster Ray) + reverse_proxy [ip_address]:7000 + + log { + output file /var/log/caddy/vllm_access.log + } +} diff --git a/containers/vllm/vllm.container b/containers/vllm/vllm.container index d107acb..12d578e 100644 --- a/containers/vllm/vllm.container +++ b/containers/vllm/vllm.container @@ -51,6 +51,15 @@ Environment=GPU_MEMORY_UTILIZATION=0.75 Environment=TENSOR_PARALLEL_SIZE=2 Environment=DTYPE=float16 Environment=KV_CACHE_DTYPE=fp8 +Environment=DISTRIBUTED_TIMEOUT_SECONDS=1800 +# Default vLLM sceglie ROCM_ATTN; scommenta per forzare il backend Triton se persistono i crash con ROCM_ATTN. +#Environment=VLLM_ATTENTION_BACKEND=TRITON_ATTN + +# Tool/function calling (richiesto dai framework di agenti per il tool_choice="auto"). +# Parser registrato in vLLM per MiniMax M2: "minimax_m2". +Environment=ENABLE_AUTO_TOOL_CHOICE=1 +Environment=TOOL_CALL_PARSER=minimax_m2 +Environment=REASONING_PARSER=minimax_m2 # Riduce la frammentazione di memoria HIP Environment=PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True