diff --git a/containers/qwen-tts/entrypoint.sh b/containers/qwen-tts/entrypoint.sh new file mode 100644 index 0000000..00b7056 --- /dev/null +++ b/containers/qwen-tts/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# Qwen3-TTS entrypoint script +# Starts the FastAPI server; model weights are downloaded automatically +# on first request via from_pretrained() and cached under /app/models +set -e + +LOAD="${QWEN_TTS_LOAD:-customvoice,voiceclone}" + +echo "=== Qwen3-TTS Server (CPU) ===" +echo "Loading: $LOAD" +echo "Device: cpu" +echo + +exec uvicorn server:app --host 0.0.0.0 --port 8000 --app-dir /app "$@" diff --git a/containers/qwen-tts/qwen-tts-cpu.Containerfile b/containers/qwen-tts/qwen-tts-cpu.Containerfile new file mode 100644 index 0000000..796e504 --- /dev/null +++ b/containers/qwen-tts/qwen-tts-cpu.Containerfile @@ -0,0 +1,39 @@ +### Qwen3-TTS Container - CPU only +### Text-to-Speech using Qwen3-TTS: https://github.com/QwenLM/Qwen3-TTS +### +### BUILD: podman build -t qwen-tts:cpu -f qwen-tts-cpu.Containerfile . +### RUN: podman run --rm -p 8000:8000 -v /path/to/models:/app/models qwen-tts:cpu +FROM python:3.12-slim + +USER root +EXPOSE 8000 + +# sox (system tool, required at runtime by qwen-tts for audio processing) +# ffmpeg (optional but commonly needed for reference-audio format conversion) +RUN apt-get update \ + && apt-get install -y --no-install-recommends sox ffmpeg curl \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Install CPU-only PyTorch first (avoids pulling CUDA wheels as a transitive dependency) +RUN pip install --no-cache-dir torch torchaudio --index-url https://download.pytorch.org/whl/cpu + +# Install qwen-tts and a minimal API server +RUN pip install --no-cache-dir qwen-tts fastapi uvicorn python-multipart + +# Models are downloaded at runtime into this directory (mount as a volume +# to persist them across container restarts and avoid re-downloading). +RUN mkdir -p /app/models +ENV HF_HOME=/app/models +ENV QWEN_TTS_DEVICE=cpu + +# Copy entrypoint / server script +COPY entrypoint.sh /app/entrypoint.sh +COPY server.py /app/server.py +RUN chmod +x /app/entrypoint.sh + +ENTRYPOINT ["/app/entrypoint.sh"] +CMD [] diff --git a/containers/qwen-tts/qwen-tts.container b/containers/qwen-tts/qwen-tts.container new file mode 100644 index 0000000..0d79a49 --- /dev/null +++ b/containers/qwen-tts/qwen-tts.container @@ -0,0 +1,39 @@ +[Unit] +Description=Qwen3-TTS Server (CPU) +After=network-online.target +Wants=network-online.target + +[Container] +Image=localhost/qwen-tts:cpu +ContainerName=qwen-tts + +# HTTP API +PublishPort=8000:8000 + +# Persist downloaded model weights across restarts +# Adjust the host path to wherever you want the models cached +Volume=/srv/containers/qwen-tts/models:/app/models:Z + +# Which model(s) to load at startup: "customvoice", "voiceclone", or both +# (default, comma-separated). Loading both lets a single instance serve +# /speech (preset voices) and /speech/clone (voice cloning) at once. +#Environment=QWEN_TTS_LOAD=customvoice,voiceclone +# Optional: override the default checkpoints +#Environment=QWEN_TTS_CUSTOMVOICE_MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice +#Environment=QWEN_TTS_VOICECLONE_MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-Base + + +# CPU-only container: no GPU devices needed +PodmanArgs=--pids-limit=-1 + +# Resource limits (tune to your host; a 1.7B model on CPU needs headroom) +#Memory=8g +#CPUs=4 + +[Service] +Restart=on-failure +RestartSec=5 +TimeoutStartSec=300 + +[Install] +WantedBy=default.target diff --git a/containers/qwen-tts/server.py b/containers/qwen-tts/server.py new file mode 100644 index 0000000..b8b3034 --- /dev/null +++ b/containers/qwen-tts/server.py @@ -0,0 +1,180 @@ +""" +Minimal FastAPI server exposing Qwen3-TTS generation over HTTP. +CPU-only: models are loaded once at startup with device_map="cpu". + +Loads BOTH Qwen3-TTS model families at once, so a single container/process +can serve: +- /speech -> CustomVoice model, 9 built-in preset speakers + (generate_custom_voice) +- /speech/clone -> Base model, clone a voice from a short reference audio + file (generate_voice_clone), 10-20s recommended, + 3s min, 60s max + +Set QWEN_TTS_LOAD=customvoice,voiceclone (default: both) to control which +model(s) are loaded, useful if you want a lighter single-purpose instance +for memory-constrained setups. Requesting an endpoint whose model wasn't +loaded returns a clear 400 error instead of a generic failure. +""" +import io +import os +import tempfile + +import soundfile as sf +import torch +from fastapi import FastAPI, File, Form, HTTPException, UploadFile +from fastapi.responses import StreamingResponse +from pydantic import BaseModel +from qwen_tts import Qwen3TTSModel + +_DEFAULT_MODELS = { + "customvoice": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "voiceclone": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", +} + +_load_env = os.environ.get("QWEN_TTS_LOAD", "customvoice,voiceclone") +LOAD_MODES = [m.strip() for m in _load_env.split(",") if m.strip()] +for m in LOAD_MODES: + if m not in _DEFAULT_MODELS: + raise ValueError( + f"QWEN_TTS_LOAD entries must be 'customvoice' and/or 'voiceclone', got: {m}" + ) + +CUSTOMVOICE_MODEL_NAME = os.environ.get( + "QWEN_TTS_CUSTOMVOICE_MODEL", _DEFAULT_MODELS["customvoice"] +) +VOICECLONE_MODEL_NAME = os.environ.get( + "QWEN_TTS_VOICECLONE_MODEL", _DEFAULT_MODELS["voiceclone"] +) + +app = FastAPI(title="Qwen3-TTS CPU server") + +# Populated at startup, keyed by mode ("customvoice" / "voiceclone") +models: dict[str, Qwen3TTSModel] = {} + + +class SpeechRequest(BaseModel): + text: str + language: str = "Auto" + speaker: str = "Vivian" + instruct: str = "" + + +def _require_model(mode: str) -> Qwen3TTSModel: + model = models.get(mode) + if model is None: + raise HTTPException( + status_code=400, + detail=f"The '{mode}' model is not loaded on this instance " + f"(QWEN_TTS_LOAD={','.join(LOAD_MODES)}). " + f"Restart the container with QWEN_TTS_LOAD including " + f"'{mode}' to use this endpoint.", + ) + return model + + +@app.on_event("startup") +def load_models(): + if "customvoice" in LOAD_MODES: + print(f"Loading CustomVoice model {CUSTOMVOICE_MODEL_NAME} on CPU...") + models["customvoice"] = Qwen3TTSModel.from_pretrained( + CUSTOMVOICE_MODEL_NAME, + device_map="cpu", + dtype=torch.float32, # bfloat16/float16 are not well supported on CPU + ) + print("CustomVoice model loaded.") + + if "voiceclone" in LOAD_MODES: + print(f"Loading Base (voice-clone) model {VOICECLONE_MODEL_NAME} on CPU...") + models["voiceclone"] = Qwen3TTSModel.from_pretrained( + VOICECLONE_MODEL_NAME, + device_map="cpu", + dtype=torch.float32, + ) + print("Base (voice-clone) model loaded.") + + +@app.get("/health") +def health(): + return { + "status": "ok", + "loaded_models": { + mode: (CUSTOMVOICE_MODEL_NAME if mode == "customvoice" else VOICECLONE_MODEL_NAME) + for mode in models + }, + } + + +@app.get("/speakers") +def speakers(): + model = _require_model("customvoice") + return {"speakers": model.get_supported_speakers()} + + +@app.post("/speech") +def speech(req: SpeechRequest): + model = _require_model("customvoice") + + wavs, sr = model.generate_custom_voice( + text=req.text, + language=req.language, + speaker=req.speaker, + instruct=req.instruct or None, + ) + + buf = io.BytesIO() + sf.write(buf, wavs[0], sr, format="WAV") + buf.seek(0) + return StreamingResponse(buf, media_type="audio/wav") + + +@app.post("/speech/clone") +def speech_clone( + text: str = Form(...), + language: str = Form("Auto"), + ref_text: str = Form(""), + x_vector_only_mode: bool = Form(False), + ref_audio: UploadFile = File(...), +): + """ + Clone a voice from a short reference audio file and synthesize `text` + with it. + + ref_audio: 10-20s of clear, continuous speech recommended (min 3s, max 60s). + ref_text: transcript of what is said in ref_audio. Required unless + x_vector_only_mode=true. + x_vector_only_mode: if true, clones only from the speaker embedding + (x-vector) without needing ref_text. Faster, but generally + lower fidelity than the default in-context-learning mode. + """ + model = _require_model("voiceclone") + + if not x_vector_only_mode and not ref_text: + raise HTTPException( + status_code=422, + detail="ref_text is required unless x_vector_only_mode=true.", + ) + + ref_bytes = ref_audio.file.read() + + # generate_voice_clone does not accept an in-memory BytesIO object, + # it expects a file path. Write the upload to a temp file instead. + suffix = os.path.splitext(ref_audio.filename or "")[1] or ".wav" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp: + tmp.write(ref_bytes) + tmp_path = tmp.name + + try: + wavs, sr = model.generate_voice_clone( + text=text, + language=language, + ref_audio=tmp_path, + ref_text=ref_text or None, + x_vector_only_mode=x_vector_only_mode, + ) + finally: + os.remove(tmp_path) + + buf = io.BytesIO() + sf.write(buf, wavs[0], sr, format="WAV") + buf.seek(0) + return StreamingResponse(buf, media_type="audio/wav") diff --git a/containers/sdcpp/build-sdcpp-vulkan.sh b/containers/sdcpp/build-sdcpp-vulkan.sh new file mode 100644 index 0000000..b062c33 --- /dev/null +++ b/containers/sdcpp/build-sdcpp-vulkan.sh @@ -0,0 +1,166 @@ +#!/bin/bash +# Build stable-diffusion.cpp with Vulkan support and create container image +# Usage: ./build-container.sh [--no-cache] + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TMP_BUILD_DIR="${SCRIPT_DIR}/.tmp" +BUILD_DIR="${TMP_BUILD_DIR}/sdcpp" +BIN_DIR="${TMP_BUILD_DIR}/bin-vulkan" +HOME_DIR="${HOME}" +NO_CACHE="" + +cleanup() { + local exit_code=$? + if [ $exit_code -ne 0 ]; then + echo + echo "⚠ Build failed (exit code: $exit_code). Cleaning up temporary files..." + fi + rm -rf "$TMP_BUILD_DIR" "${SCRIPT_DIR}/bin-vulkan" 2>/dev/null || true + if [ $exit_code -eq 0 ]; then + echo "✓ Cleanup complete" + fi + return $exit_code +} +trap cleanup EXIT + +if [[ "$1" == "--no-cache" ]]; then + NO_CACHE="--no-cache" + echo "Build mode: NO CACHE (clean rebuild)" +fi + +echo "=== stable-diffusion.cpp Vulkan Build ===" +echo "Temporary build directory: $TMP_BUILD_DIR" +echo "Output directory: $BIN_DIR" +echo "Home directory: $HOME_DIR" +if [ -n "$NO_CACHE" ]; then + echo "Cache mode: DISABLED" +fi +echo + +mkdir -p "$TMP_BUILD_DIR" +echo + +# Install dependencies +echo "[1/5] Installing dependencies..." +REQUIRED_PACKAGES="build-essential cmake git libvulkan-dev glslc spirv-headers" +MISSING_PACKAGES="" + +for pkg in $REQUIRED_PACKAGES; do + if ! dpkg -l | grep -q "^ii $pkg"; then + MISSING_PACKAGES="$MISSING_PACKAGES $pkg" + fi +done + +if [ -n "$MISSING_PACKAGES" ]; then + echo " Installing missing packages:$MISSING_PACKAGES" + sudo apt-get update + sudo apt-get install -y $MISSING_PACKAGES + echo "✓ Dependencies installed" +else + echo "✓ All dependencies already installed" +fi + +echo " Verifying dependencies..." +for cmd in git cmake make gcc g++ glslc; do + if ! command -v $cmd &> /dev/null; then + echo "ERROR: $cmd is still not available after installation." + exit 1 + fi +done + +if ! pkg-config --exists vulkan; then + echo "ERROR: Vulkan development files not found." + exit 1 +fi + +echo "✓ All dependencies verified" +echo + +# Clone or update stable-diffusion.cpp (with submodules: ggml is vendored as a submodule) +echo "[2/5] Cloning/updating stable-diffusion.cpp repository..." +if [ -d "$BUILD_DIR" ]; then + echo " Updating existing repository..." + cd "$BUILD_DIR" + git fetch origin + git checkout master + git pull origin master + git submodule update --init --recursive +else + echo " Cloning stable-diffusion.cpp (with submodules)..." + git clone --recursive https://github.com/leejet/stable-diffusion.cpp.git "$BUILD_DIR" + cd "$BUILD_DIR" +fi +echo "✓ Repository ready" +echo + +# Build with Vulkan support +echo "[3/5] Building with Vulkan support (this may take a while)..." +mkdir -p build +cd build + +if [ -n "$NO_CACHE" ]; then + echo " Cleaning previous build..." + rm -rf * .cmake +fi + +cmake .. -DSD_VULKAN=ON -DCMAKE_BUILD_TYPE=Release +cmake --build . --config Release -j "$(nproc)" +echo "✓ Build complete" +echo + +# Prepare binary directory +echo "[4/5] Preparing binary directory..." +mkdir -p "$BIN_DIR" +cd "$BUILD_DIR" + +echo " Copying binaries..." +# sd-cli is the current binary name (renamed from the old 'sd' in upstream #1037) +cp build/bin/sd-cli "$BIN_DIR/" +chmod +x "$BIN_DIR"/* +echo "✓ Binaries ready" +echo + +# Build Podman image +echo "[5/5] Building Podman image..." + +echo " Preparing build context..." +cp "${SCRIPT_DIR}/sdcpp-vulkan.Containerfile" "${TMP_BUILD_DIR}/" +cp "${SCRIPT_DIR}/sdcpp-entrypoint.sh" "${TMP_BUILD_DIR}/entrypoint.sh" +cp "${SCRIPT_DIR}/watch-gpu.sh" "${TMP_BUILD_DIR}/" + +cd "$TMP_BUILD_DIR" +podman build $NO_CACHE -t sdcpp:vulkan-amd64 -f sdcpp-vulkan.Containerfile . +echo "✓ Podman image built" +echo + +echo "=== BUILD COMPLETE ===" +echo +echo "✓ Container image created: sdcpp:vulkan-amd64" +echo +echo "Next steps:" +echo " 1. Create a models directory with the GGUF files, e.g.:" +echo " mkdir -p ~/sdcpp-models" +echo " # place minimax_h3_fl2va_pruned-*.gguf, qwen3vl_32b_minimax_h3-*.gguf," +echo " # minimax_h3_video_vae_fp16.safetensors, minimax_h3_audio_vae_fp32.safetensors there" +echo +echo " 2. Run directly:" +echo " podman run --rm -it \\" +echo " --device /dev/dri/renderD128 \\" +echo " --group-add keep-groups \\" +echo " -v ~/sdcpp-models:/app/models \\" +echo " -v ~/sdcpp-output:/app/output \\" +echo " sdcpp:vulkan-amd64 \\" +echo " --mode vid_gen \\" +echo " --diffusion-model /app/models/minimax_h3_fl2va_pruned-Q4_K.gguf \\" +echo " --llm /app/models/qwen3vl_32b_minimax_h3-Q4_K_M.gguf \\" +echo " --vae /app/models/minimax_h3_video_vae_fp16.safetensors \\" +echo " --audio-vae /app/models/minimax_h3_audio_vae_fp32.safetensors \\" +echo " --prompt \"a red fox trotting through falling snow, cinematic\" \\" +echo " --width 640 --height 384 --video-frames 25 --steps 4 --cfg-scale 1.0 \\" +echo " --backend te=cpu --diffusion-fa \\" +echo " --output /app/output/out.webm" +echo +echo " 3. Or install as a Quadlet service (see sdcpp.container)" +echo diff --git a/containers/sdcpp/sdcpp-entrypoint.sh b/containers/sdcpp/sdcpp-entrypoint.sh new file mode 100644 index 0000000..9618305 --- /dev/null +++ b/containers/sdcpp/sdcpp-entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# stable-diffusion.cpp entrypoint +# Starts the optional GPU watchdog in background (best-effort, never fatal), +# then runs sd-cli with whatever arguments were passed to the container. +set -e + +echo "=== stable-diffusion.cpp (Vulkan) ===" +echo + +# Start the watchdog in the background if /dev/kmsg is accessible. +# It only prints warnings to stderr, it never interferes with sd-cli. +/app/watch-gpu.sh /dev/kmsg & +WATCH_PID=$! +disown "$WATCH_PID" 2>/dev/null || true + +if [ "$#" -eq 0 ]; then + echo "No arguments passed. Example usage:" + echo + echo " --mode vid_gen \\" + echo " --diffusion-model /app/models/minimax_h3_fl2va_pruned-Q4_K.gguf \\" + echo " --llm /app/models/qwen3vl_32b_minimax_h3-Q4_K_M.gguf \\" + echo " --vae /app/models/minimax_h3_video_vae_fp16.safetensors \\" + echo " --audio-vae /app/models/minimax_h3_audio_vae_fp32.safetensors \\" + echo " --prompt \"...\" --width 640 --height 384 --video-frames 25 \\" + echo " --steps 4 --cfg-scale 1.0 --backend te=cpu --diffusion-fa \\" + echo " --output /app/output/out.webm" + exec /app/sd-cli --help +fi + +exec /app/sd-cli "$@" diff --git a/containers/sdcpp/sdcpp-vulkan.Containerfile b/containers/sdcpp/sdcpp-vulkan.Containerfile new file mode 100644 index 0000000..64e8aaa --- /dev/null +++ b/containers/sdcpp/sdcpp-vulkan.Containerfile @@ -0,0 +1,41 @@ +### stable-diffusion.cpp Container with Vulkan GPU support +### Diffusion model (image/video/audio) inference: https://github.com/leejet/stable-diffusion.cpp +### +### BUILD: ./build-sdcpp-vulkan.sh (compiles locally with Vulkan) +### THEN: podman build -t sdcpp:vulkan-amd64 -f sdcpp-vulkan.Containerfile . +FROM debian:13-slim + +USER root + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libvulkan1 vulkan-tools mesa-vulkan-drivers libdrm-amdgpu1 \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /tmp/* /var/tmp/* \ + && rm -rf /var/lib/apt/lists/* \ + && find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \ + && find /var/cache -type f -delete + +WORKDIR /app + +# Copy pre-compiled binary with Vulkan support +COPY bin-vulkan/ /app/ +RUN chmod +x /app/sd-cli + +# Copy entrypoint and GPU-crash watchdog helper +COPY entrypoint.sh /app/entrypoint.sh +COPY watch-gpu.sh /app/watch-gpu.sh +RUN chmod +x /app/entrypoint.sh /app/watch-gpu.sh + +# Models and output are mounted as volumes at runtime +RUN mkdir -p /app/models /app/output + +ENV PATH=/app:$PATH +ENV LD_LIBRARY_PATH=/app:/usr/local/lib:/usr/lib:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +# Vulkan environment (adjust if your ICD lives elsewhere) +ENV VK_DRIVER_FILES=/usr/share/vulkan/icd.d/radeon_icd.json + +WORKDIR /app +ENTRYPOINT ["/app/entrypoint.sh"] +CMD [] diff --git a/containers/sdcpp/sdcpp.container b/containers/sdcpp/sdcpp.container new file mode 100644 index 0000000..6111813 --- /dev/null +++ b/containers/sdcpp/sdcpp.container @@ -0,0 +1,40 @@ +[Unit] +Description=stable-diffusion.cpp (Vulkan, video generation) +After=network-online.target +Wants=network-online.target + +[Container] +Image=localhost/sdcpp:vulkan-amd64 +ContainerName=sdcpp-vulkan + +# GPU access (Vulkan render node) +AddDevice=/dev/dri/renderD128 +PodmanArgs=--group-add=keep-groups + +# Let the in-container watchdog read the kernel ring buffer to spot +# amdgpu MES hangs early. Optional: comment out if you'd rather not +# expose /dev/kmsg to the container. +AddDevice=/dev/kmsg + +# Models (GGUF denoiser + text encoder + VAEs) and generated output. +# Populate %h/sdcpp-models with the files from the model card before +# starting a generation run. +Volume=%h/sdcpp-models:/app/models:Z +Volume=%h/sdcpp-output:/app/output:Z + +# This is meant to be run on-demand with specific --prompt / --output +# arguments rather than as a long-running daemon, so Exec is left to +# `podman start --attach` invocations with Exec= overrides, e.g.: +# systemctl --user start sdcpp-vulkan +# then attach and pass args via `podman exec` is not applicable for +# one-shot CLI tools; prefer running with `podman run` directly for +# ad-hoc generations, or set Exec below for a fixed recurring job. +#Exec=--mode vid_gen --diffusion-model /app/models/minimax_h3_fl2va_pruned-Q4_K.gguf --llm /app/models/qwen3vl_32b_minimax_h3-Q4_K_M.gguf --vae /app/models/minimax_h3_video_vae_fp16.safetensors --audio-vae /app/models/minimax_h3_audio_vae_fp32.safetensors --prompt "a red fox trotting through falling snow, cinematic" --width 640 --height 384 --video-frames 25 --steps 4 --cfg-scale 1.0 --backend te=cpu --diffusion-fa --output /app/output/out.webm + +[Service] +Type=oneshot +RemainAfterExit=no +TimeoutStartSec=1800 + +[Install] +WantedBy=default.target diff --git a/containers/sdcpp/watch-gpu.sh b/containers/sdcpp/watch-gpu.sh new file mode 100644 index 0000000..465d410 --- /dev/null +++ b/containers/sdcpp/watch-gpu.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Lightweight GPU watchdog: tails the host kernel ring buffer (if mounted) +# and warns loudly if the known amdgpu MES-hang pattern shows up while +# sd-cli is generating. Purely informational: it never kills the process, +# it just makes the problem visible immediately instead of after the fact. +# +# Requires /dev/kmsg (or a mounted /host-dmesg log) to be readable inside +# the container. If it isn't available, this script exits quietly. + +set -u + +LOGFILE="${1:-/dev/kmsg}" + +if [ ! -r "$LOGFILE" ]; then + echo "[watch-gpu] $LOGFILE not readable from inside the container, skipping GPU watchdog." >&2 + exit 0 +fi + +echo "[watch-gpu] Monitoring $LOGFILE for amdgpu MES/reset issues..." + +tail -F -n0 "$LOGFILE" 2>/dev/null | while read -r line; do + case "$line" in + *"MES failed to respond"*|*"GPU reset"*|*"ring"*"timeout"*|*"device wedged"*|*"soft lockup"*) + echo "[watch-gpu] !!! GPU issue detected: $line" >&2 + ;; + esac +done diff --git a/containers/talkbot/README.md b/containers/talkbot/README.md new file mode 100644 index 0000000..dfeb8c7 --- /dev/null +++ b/containers/talkbot/README.md @@ -0,0 +1,208 @@ +# Nextcloud Talk voice-translate bot + +Bot per Nextcloud Talk: quando arriva un messaggio vocale in una conversazione +dove il bot è presente, lo trascrive, lo traduce nella lingua preferita del +destinatario e risponde con il testo tradotto (l'audio sintetizzato verrà +allegato una volta completata l'integrazione con l'upload file, vedi TODO). + +## Architettura + +``` +Nextcloud Talk (webhook bot, bots-v1) + → talkbot/server.py (questo container) + → whisper.cpp : trascrizione audio → testo + lingua rilevata + → llama.cpp : traduzione testo → lingua del destinatario + → qwen-tts : sintesi vocale del testo tradotto + - se esiste voice-samples/.wav → voce clonata + (istanza qwen-tts in modalità voiceclone) + - altrimenti → voce preset di default (istanza qwen-tts in + modalità customvoice) + → talkbot/server.py posta la risposta in chat (bot message API) +``` + +Il container è **generico e riusabile**: lo stesso bot (stesso secret) può +essere aggiunto a più conversazioni Talk. Il token della conversazione +arriva in ogni richiesta webhook (`target.id`), quindi non serve +un'istanza per chat. + +## Voce personalizzata per utente + +Se in `VOICE_SAMPLES_DIR` (di default `/app/voice-samples`, montato da +`~/talkbot-voice-samples` via Quadlet) esiste un file chiamato +`.wav`, il bot userà quella voce (voice cloning) invece +della voce preimpostata di default. + +`` è lo user ID Nextcloud del **mittente** del messaggio +vocale (lo stesso usato per determinare la lingua di destinazione). + +**Qualità del cloning**: puoi opzionalmente affiancare al campione un +file `.txt` con la trascrizione esatta di cosa dice il +`.wav` (es. `sara.wav` + `sara.txt`). Se presente, il bot userà la +modalità ICL (in-context learning) di qwen-tts, che dà una clonazione più +fedele. Se manca il `.txt`, si usa automaticamente `x_vector_only_mode` +(più veloce, qualità leggermente inferiore, non richiede trascrizione). + +Il testo di riferimento puoi ottenerlo facilmente trascrivendo tu stesso +il campione con whisper: +```bash +curl -X POST http://localhost:8080/inference \ + -F "file=@sara.wav" -F "response_format=json" +``` +e salvando il campo `text` risultante in `sara.txt`. + +Il container qwen-tts carica **entrambi i modelli** (CustomVoice e Base) +nello stesso processo (`QWEN_TTS_LOAD=customvoice,voiceclone`, default), +così un'unica istanza espone sia `/speech` (voce preset) che +`/speech/clone` (voce clonata): il bot punta a un solo `QWEN_TTS_URL` e +sceglie l'endpoint giusto in base alla presenza del campione utente. +Questo costa più RAM/tempo di avvio rispetto a un solo modello, ma evita +di dover gestire due container qwen-tts separati. + +## File + +| File | Scopo | +| ------------------------ | ------------------------------------------------------------------ | +| `server.py` | Logica del bot: webhook, pipeline whisper→llama.cpp→qwen-tts, risposta | +| `entrypoint.sh` | Avvia uvicorn, valida le variabili d'ambiente richieste | +| `talkbot.Containerfile` | Immagine del container | +| `talkbot.container` | Quadlet Podman/systemd per l'esecuzione come servizio | +| `README.md` | Questo file | + +## Setup + +### 1. Registrare il bot su Nextcloud + +Serve accesso CLI (`occ`) al server Nextcloud: + +```bash +occ talk:bot:install "TranslateBot" https://talkbot.example.tld/webhook \ + --feature=webhook +``` + +`` deve corrispondere a `NC_BOT_SECRET` nel Quadlet. +L'URL deve essere raggiungibile dal server Nextcloud (attenzione a +firewall/reti interne se Nextcloud e il bot non sono sulla stessa rete). + +Per aggiungere il bot a una conversazione specifica, va abilitato dalle +impostazioni della conversazione stessa (icona bot) o via API conversazioni. + +### 2. App password Nextcloud + +`NC_ADMIN_USER` / `NC_ADMIN_PASSWORD` sono usate per: +- leggere la lingua preferita del mittente (`/ocs/v1.php/cloud/users/{id}`) +- scaricare l'allegato audio del messaggio vocale +- caricare l'audio tradotto generato (upload lato Files) + +Usa una **app password** dedicata (Impostazioni personali → Sicurezza → +Password per app), non la password reale dell'account. + +### 3. Build e avvio + +```bash +podman build -t talkbot:latest -f talkbot.Containerfile . +cp talkbot.container ~/.config/containers/systemd/ +# modifica i valori NC_URL / NC_BOT_SECRET / NC_ADMIN_USER / NC_ADMIN_PASSWORD +# e gli URL di whisper/llama.cpp/qwen-tts nel file .container +systemctl --user daemon-reload +systemctl --user start talkbot +journalctl --user -u talkbot -f +``` + +### 4. Test rapido + +```bash +curl http://localhost:8100/health +# {"status":"ok"} +``` + +Il vero test end-to-end richiede un messaggio vocale reale inviato in una +conversazione Talk dove il bot è abilitato: vedi sezione "Stato dei test". + +## Variabili d'ambiente + +| Variabile | Obbligatoria | Default | Note | +| -------------------------- | ------------ | ------------------- | ------------------------------------------------- | +| `NC_URL` | sì | — | URL base Nextcloud, senza slash finale | +| `NC_BOT_SECRET` | sì | — | Secret condiviso usato in `occ talk:bot:install` | +| `NC_ADMIN_USER` | sì | — | Utente Nextcloud per API/download/upload | +| `NC_ADMIN_PASSWORD` | sì | — | App password di `NC_ADMIN_USER` | +| `WHISPER_URL` | no | `http://whisper:8080` | Endpoint whisper-server | +| `LLAMACPP_URL` | no | `http://llamacpp:7000` | Endpoint llama.cpp (OpenAI-compatible chat) | +| `LLAMACPP_MODEL` | no | (vuoto) | Nome modello, se il tuo router llama.cpp lo richiede | +| `LLAMACPP_TIMEOUT` | no | `600` | Timeout (secondi) per la chiamata a llama.cpp | +| `QWEN_TTS_URL` | no | `http://qwen-tts:8000` | Endpoint qwen-tts (deve girare con `QWEN_TTS_LOAD` includendo sia `customvoice` che `voiceclone`, default) | +| `QWEN_TTS_TIMEOUT` | no | `1800` | Timeout (secondi) per la chiamata a qwen-tts | +| `VOICE_SAMPLES_DIR` | no | `/app/voice-samples` | Cartella con i campioni `.wav` per il cloning | +| `DEFAULT_TARGET_LANGUAGE` | no | `en` | Lingua di fallback se non si riesce a leggere quella dell'utente | +| `QWEN_TTS_SPEAKER` | no | `Ryan` | Voce preset usata quando non c'è un campione utente | + +## Stato dei test / cosa manca ancora + +Questo file va aggiornato man mano che verifichiamo il comportamento reale +contro un'istanza Nextcloud Talk vera. Alla creazione (2 settembre 2026): + +- [ ] **Non ancora testato contro un webhook reale.** La struttura del + payload per i messaggi vocali (dove si trova esattamente l'URL/ID del + file allegato dentro `object.content.parameters`) è assunta dalla + documentazione generica dei bot, ma va confermata con un vero + messaggio vocale: il parametro potrebbe chiamarsi diversamente o + avere una struttura leggermente diversa da quella ipotizzata in + `file_param` dentro `server.py`. +- [ ] **Download dell'allegato audio**: `download_attachment()` assume che + `file_param["link"]` sia un URL scaricabile direttamente con + Basic Auth. Da verificare se serve invece passare dal WebDAV + (`/remote.php/dav/files/...`) o da un endpoint di preview/download + specifico dei messaggi di chat. +- [ ] **Upload e allegazione dell'audio tradotto**: `upload_and_share_audio()` + è un placeholder. Attualmente il bot risponde solo con il **testo** + tradotto (via `send_bot_message`), non ancora con il file audio + allegato in chat. Serve capire l'API esatta di Talk per condividere + un file già presente nei Files dell'utente dentro una conversazione + (probabilmente `POST /ocs/v2.php/apps/spreed/api/v1/chat/{token}/share` + o simile — da verificare nella doc "Chat management"). +- [ ] **Lingua in chat di gruppo**: `get_user_language(actor_id)` prende la + lingua di chi ha *inviato* il messaggio, non del destinatario. Per + chat 1:1 o per un bot dedicato a un singolo utente va bene così; per + gruppi con più utenti/lingue diverse la logica "traduci nella lingua + del destinatario" va ripensata (quale destinatario? tutti? uno alla + volta?). +- [ ] **Rilevamento lingua sorgente**: whisper-server deve essere lanciato + con `WHISPER_LANGUAGE=auto` (o comunque senza lingua fissa) perché + `detected_lang` in `transcribe()` sia affidabile. +- [ ] **Gestione errori/timeout**: i tre servizi (whisper, llama.cpp, + qwen-tts) sono chiamati in sequenza sincrona nello stesso worker + HTTP; su CPU la sintesi qwen-tts può richiedere decine di secondi. + Da valutare se serve un timeout più alto lato Nextcloud o passare a + elaborazione asincrona con risposta differita. +- [x] ~~Doppia istanza qwen-tts~~: risolto, qwen-tts ora carica entrambi i + modelli (CustomVoice + Base) nello stesso processo via + `QWEN_TTS_LOAD=customvoice,voiceclone` (default). Un solo container, + un solo `QWEN_TTS_URL`, entrambi gli endpoint disponibili. **Nota**: + questo raddoppia la RAM e il tempo di avvio rispetto a un solo + modello (due checkpoint da 1.7B caricati in memoria su CPU), tienine + conto se il sistema ha risorse limitate — in tal caso si può + restringere a un solo modello con `QWEN_TTS_LOAD=customvoice` (o + `voiceclone`), sapendo che l'endpoint mancante risponderà 400. +- [ ] **Provenienza dei campioni voce**: al momento i file + `voice-samples/.wav` vanno creati manualmente + sull'host (es. copiandoli dal container qwen-tts o da altre + registrazioni). Non c'è ancora un modo per l'utente di caricare il + proprio campione dalla chat stessa (es. "invia un vocale e digli + /set-voice per usarlo come tuo campione"). +- [ ] **Sicurezza rete**: il Quadlet pubblica la porta 8100 su tutte le + interfacce; se Nextcloud non è sulla stessa rete Podman/host, va + esposto solo dove necessario (reverse proxy, firewall). + +## Note di debug utili + +```bash +# Log del bot +journalctl --user -u talkbot -f + +# Verifica manuale della pipeline, bypassando Nextcloud: +curl -X POST http://localhost:8080/inference -F "file=@test.wav" -F "response_format=json" +curl -X POST http://localhost:7000/v1/chat/completions -H "Content-Type: application/json" \ + -d '{"messages":[{"role":"user","content":"Translate into Italian: Hello"}]}' +curl -X POST http://localhost:8000/speech -H "Content-Type: application/json" \ + -d '{"text":"Ciao","language":"Italian","speaker":"Ryan"}' --output test-tts.wav +``` diff --git a/containers/talkbot/entrypoint.sh b/containers/talkbot/entrypoint.sh new file mode 100644 index 0000000..d819e80 --- /dev/null +++ b/containers/talkbot/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Nextcloud Talk voice-translate bot entrypoint +set -e + +echo "=== Nextcloud Talk voice-translate bot ===" +echo "NC_URL: ${NC_URL:-}" +echo "WHISPER_URL: ${WHISPER_URL:-http://whisper:8080}" +echo "LLAMACPP_URL: ${LLAMACPP_URL:-http://llamacpp:7000}" +echo "QWEN_TTS_URL: ${QWEN_TTS_URL:-http://qwen-tts:8000}" +echo "Default lang: ${DEFAULT_TARGET_LANGUAGE:-en}" +echo + +for required in NC_URL NC_BOT_SECRET NC_USER NC_PASSWORD; do + if [ -z "${!required}" ]; then + echo "ERROR: required environment variable $required is not set." + exit 1 + fi +done + +exec uvicorn server:app --host 0.0.0.0 --port 8100 --app-dir /app "$@" diff --git a/containers/talkbot/server.py b/containers/talkbot/server.py new file mode 100644 index 0000000..57fbc3b --- /dev/null +++ b/containers/talkbot/server.py @@ -0,0 +1,1191 @@ +import json +import logging +import os +import re +from datetime import datetime +from typing import Optional + +import httpx +from fastapi import FastAPI, Header, HTTPException, Request +from fastapi.responses import JSONResponse + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", +) +log = logging.getLogger("talkbot") + +NC_URL = os.environ["NC_URL"].rstrip("/") +NC_USER = os.environ["NC_USER"] +NC_PASSWORD = os.environ["NC_PASSWORD"] + +WHISPER_URL = os.environ.get("WHISPER_URL", "http://whisper:8080").rstrip("/") +LLAMACPP_URL = os.environ.get("LLAMACPP_URL", "http://llamacpp:7000").rstrip("/") +LLAMACPP_MODEL = os.environ.get("LLAMACPP_MODEL", "").strip() +LLAMACPP_TIMEOUT = int(os.environ.get("LLAMACPP_TIMEOUT", "600")) +LLAMACPP_ENABLE_THINKING = ( + os.environ.get("LLAMACPP_ENABLE_THINKING", "false").lower() + in {"1", "true", "yes", "on"} +) +QWEN_TTS_URL = os.environ.get("QWEN_TTS_URL", "http://qwen-tts:8000").rstrip("/") +QWEN_TTS_TIMEOUT = int(os.environ.get("QWEN_TTS_TIMEOUT", "1800")) + +DEFAULT_TARGET_LANGUAGE = os.environ.get( + "DEFAULT_TARGET_LANGUAGE", + "it", +).strip() + +QWEN_TTS_SPEAKER = os.environ.get( + "QWEN_TTS_SPEAKER", + "Ryan", +).strip() + +VOICE_SAMPLES_DIR = os.environ.get( + "VOICE_SAMPLES_DIR", + "/app/voice-samples", +) + +MANAGER_USER = NC_USER + +app = FastAPI(title="Nextcloud Talk voice translator") + +LANGUAGE_NAMES = { + "en": "English", + "it": "Italian", + "de": "German", + "fr": "French", + "es": "Spanish", + "pt": "Portuguese", + "nl": "Dutch", + "pl": "Polish", + "ru": "Russian", + "zh": "Chinese", + "ja": "Japanese", + "ko": "Korean", +} + +LANGUAGE_CODES = { + "en": "EN", + "it": "IT", + "de": "DE", + "fr": "FR", + "es": "ES", + "pt": "PT", + "nl": "NL", + "pl": "PL", + "ru": "RU", + "zh": "ZH", + "ja": "JA", + "ko": "KO", +} + + +def normalize_language_code(language: str) -> str: + language = (language or "").strip().lower() + + if not language: + return DEFAULT_TARGET_LANGUAGE.lower() + + if "-" in language: + language = language.split("-", 1)[0] + + if "_" in language: + language = language.split("_", 1)[0] + + return language + + +def language_name(language: str) -> str: + code = normalize_language_code(language) + return LANGUAGE_NAMES.get( + code, + language if language else LANGUAGE_NAMES.get( + normalize_language_code(DEFAULT_TARGET_LANGUAGE), + "English", + ), + ) + + +def language_short_code(language: str) -> str: + code = normalize_language_code(language) + return LANGUAGE_CODES.get(code, code.upper()[:8]) + + +def sanitize_filename_part(value: str) -> str: + value = (value or "").strip() + value = re.sub(r"[^\w.-]+", "_", value, flags=re.UNICODE) + value = value.strip("._-") + return value or "User" + + +def verify_webhook_backend( + random_header: Optional[str], + signature_header: Optional[str], + body: bytes, +) -> bool: + if not random_header or not signature_header: + return False + + import hashlib + import hmac + + secret = os.environ.get("NC_BOT_SECRET", "") + if not secret: + return True + + digest = hmac.new( + secret.encode(), + (random_header + body.decode("utf-8", errors="ignore")).encode(), + hashlib.sha256, + ).hexdigest() + + return hmac.compare_digest( + digest.lower(), + signature_header.lower(), + ) + + +def get_user_language(user_id: str) -> str: + try: + response = httpx.get( + f"{NC_URL}/ocs/v1.php/cloud/users/{user_id}", + headers={ + "OCS-APIRequest": "true", + "Accept": "application/json", + }, + auth=(NC_USER, NC_PASSWORD), + timeout=15, + ) + + response.raise_for_status() + + data = response.json() + language = ( + data + .get("ocs", {}) + .get("data", {}) + .get("language") + ) + + if language: + return language + + except Exception as exc: + log.warning( + "Could not fetch language for user %s: %s", + user_id, + exc, + ) + + return DEFAULT_TARGET_LANGUAGE + + +def extract_file_param(parameters) -> Optional[dict]: + if isinstance(parameters, dict): + values = list(parameters.values()) + elif isinstance(parameters, list): + values = parameters + else: + values = [] + + for value in values: + if isinstance(value, dict) and value.get("type") == "file": + return value + + return None + + +def actor_user_id(actor: dict) -> str: + actor_id = str(actor.get("id", "") or "") + + if "/" in actor_id: + return actor_id.split("/", 1)[-1] + + return actor_id + + +def get_conversation_participants(token: str) -> list[dict]: + response = httpx.get( + f"{NC_URL}/ocs/v2.php/apps/spreed/api/v4/room/{token}/participants", + headers={ + "OCS-APIRequest": "true", + "Accept": "application/json", + }, + auth=(NC_USER, NC_PASSWORD), + timeout=30, + ) + + response.raise_for_status() + + data = response.json().get("ocs", {}).get("data", []) + + if isinstance(data, list): + return [item for item in data if isinstance(item, dict)] + + if isinstance(data, dict): + for key in ("participants", "data", "users"): + value = data.get(key) + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + + return [] + + +def get_target_user_ids(token: str, sender_id: str) -> list[str]: + result = [] + + for participant in get_conversation_participants(token): + actor_type = str( + participant.get("actorType") + or participant.get("type") + or "" + ).lower() + + if actor_type not in {"users", "user"}: + continue + + actor_id = str( + participant.get("actorId") + or participant.get("id") + or "" + ) + + user_id = actor_id.split("/", 1)[-1].strip() + + if not user_id: + continue + + if user_id == sender_id: + continue + + if user_id == MANAGER_USER: + continue + + if user_id not in result: + result.append(user_id) + + return result + + +def request_direct_download(file_id: int) -> str: + response = httpx.post( + f"{NC_URL}/ocs/v2.php/apps/dav/api/v1/direct", + headers={ + "OCS-APIRequest": "true", + "Accept": "application/json", + "Content-Type": "application/json", + }, + auth=(NC_USER, NC_PASSWORD), + json={ + "fileId": int(file_id), + "expirationTime": 600, + }, + timeout=30, + ) + + log.info( + "Direct download request for file %s: HTTP %s", + file_id, + response.status_code, + ) + + response.raise_for_status() + + data = response.json() + + ocs_data = data.get("ocs", {}).get("data", {}) + + url = None + + if isinstance(ocs_data, dict): + url = ( + ocs_data.get("url") + or ocs_data.get("link") + or ocs_data.get("downloadUrl") + ) + + if not url and isinstance(ocs_data, str): + url = ocs_data + + if not url: + raise RuntimeError( + f"Nextcloud did not return a direct download URL for file {file_id}" + ) + + if url.startswith("https://"): + if NC_URL.startswith("http://"): + url = url.replace("https://", "http://", 1) + + elif url.startswith("/"): + url = f"{NC_URL}{url}" + + log.info( + "Normalized Direct Download URL for file %s", + file_id, + ) + + return url + + +def download_attachment(file_id: int) -> bytes: + direct_url = request_direct_download(file_id) + + log.info( + "Downloading file through Direct Download: %s", + direct_url, + ) + + response = httpx.get( + direct_url, + timeout=120, + follow_redirects=True, + ) + + log.info( + "Direct download HTTP %s", + response.status_code, + ) + + response.raise_for_status() + + log.info( + "Downloaded audio: %s bytes", + len(response.content), + ) + + return response.content + + +def is_blank_transcription(text: str) -> bool: + normalized = (text or "").strip().lower() + if not normalized: + return True + + normalized = re.sub(r"[\s._-]+", "", normalized) + return normalized in { + "[blankaudio]", + "(blankaudio)", + "blankaudio", + "[silence]", + "(silence)", + "silence", + } + + +def transcribe(audio_bytes: bytes) -> tuple[str, str]: + response = httpx.post( + f"{WHISPER_URL}/inference", + files={ + "file": ( + "audio.wav", + audio_bytes, + "audio/wav", + ) + }, + data={ + "response_format": "json", + }, + timeout=300, + ) + + response.raise_for_status() + + result = response.json() + + text = str(result.get("text", "") or "").strip() + detected_language = str( + result.get("language", "auto") or "auto" + ).strip() + + return text, detected_language + + +def translate( + text: str, + source_language: str, + target_language: str, +) -> str: + source_name = language_name(source_language) + target_name = language_name(target_language) + + prompt = ( + f"Translate the following text from {source_name} " + f"to {target_name}.\n" + "Return only the translated text.\n" + "Do not explain the translation.\n" + "Do not answer the message.\n" + "Do not add comments.\n\n" + f"Text:\n{text}" + ) + + payload = { + "messages": [ + { + "role": "system", + "content": ( + "You are a professional translator. " + "You must translate exactly as requested." + ), + }, + { + "role": "user", + "content": prompt, + }, + ], + "temperature": 0.1, + "chat_template_kwargs": { + "enable_thinking": LLAMACPP_ENABLE_THINKING, + }, + } + + if LLAMACPP_MODEL: + payload["model"] = LLAMACPP_MODEL + + response = httpx.post( + f"{LLAMACPP_URL}/v1/chat/completions", + json=payload, + timeout=LLAMACPP_TIMEOUT, + ) + + response.raise_for_status() + + result = response.json() + + return ( + result["choices"][0]["message"]["content"] + .strip() + ) + + +def find_voice_sample( + user_id: str, +) -> tuple[Optional[str], Optional[str]]: + safe_user_id = re.sub( + r"[^A-Za-z0-9_.-]", + "", + user_id, + ) + + if not safe_user_id: + return None, None + + wav_path = os.path.join( + VOICE_SAMPLES_DIR, + f"{safe_user_id}.wav", + ) + + if not os.path.isfile(wav_path): + return None, None + + txt_path = os.path.join( + VOICE_SAMPLES_DIR, + f"{safe_user_id}.txt", + ) + + ref_text = None + + if os.path.isfile(txt_path): + with open( + txt_path, + "r", + encoding="utf-8", + ) as file: + ref_text = file.read().strip() or None + + return wav_path, ref_text + + +def synthesize( + text: str, + target_language: str, + sender_user_id: str, +) -> bytes: + target_name = language_name(target_language) + + sample_path, ref_text = find_voice_sample( + sender_user_id, + ) + + if sample_path: + log.info( + "Using cloned voice from %s for user %s", + sample_path, + sender_user_id, + ) + + data = { + "text": text, + "language": target_name, + } + + if ref_text: + data["ref_text"] = ref_text + else: + data["x_vector_only_mode"] = "true" + + with open(sample_path, "rb") as reference_audio: + response = httpx.post( + f"{QWEN_TTS_URL}/speech/clone", + data=data, + files={ + "ref_audio": ( + os.path.basename(sample_path), + reference_audio, + "audio/wav", + ) + }, + timeout=QWEN_TTS_TIMEOUT, + ) + + else: + log.info( + "No voice sample for user %s, using default speaker %s", + sender_user_id, + QWEN_TTS_SPEAKER, + ) + + response = httpx.post( + f"{QWEN_TTS_URL}/speech", + json={ + "text": text, + "language": target_name, + "speaker": QWEN_TTS_SPEAKER, + }, + timeout=QWEN_TTS_TIMEOUT, + ) + + response.raise_for_status() + + return response.content + + +def delete_message( + token: str, + message_id: int, +) -> bool: + url = ( + f"{NC_URL}/ocs/v2.php/apps/spreed/api/v1/" + f"chat/{token}/{message_id}" + ) + + response = httpx.delete( + url, + headers={ + "OCS-APIRequest": "true", + "Accept": "application/json", + }, + auth=(NC_USER, NC_PASSWORD), + timeout=30, + ) + + log.info( + "Delete message %s: HTTP %s", + message_id, + response.status_code, + ) + + if response.status_code == 200: + log.info( + "Deleted original message %s", + message_id, + ) + return True + + if response.status_code == 202: + log.info( + "Deleted original message %s: HTTP 202", + message_id, + ) + return True + + if response.status_code == 403: + log.warning( + "Manager is not allowed to delete message %s " + "(Manager must be a moderator)", + message_id, + ) + return False + + if response.status_code == 404: + log.warning( + "Message %s was not found", + message_id, + ) + return False + + log.warning( + "Could not delete message %s: HTTP %s: %s", + message_id, + response.status_code, + response.text[:500], + ) + + return False + + +def ensure_manager_folder() -> None: + response = httpx.request( + "MKCOL", + f"{NC_URL}/remote.php/dav/files/" + f"{MANAGER_USER}/talk-bot-output/", + auth=(NC_USER, NC_PASSWORD), + timeout=30, + ) + + if response.status_code not in (201, 405): + response.raise_for_status() + + +def upload_audio( + wav_bytes: bytes, + filename: str, +) -> str: + ensure_manager_folder() + + dav_path = ( + f"/remote.php/dav/files/{MANAGER_USER}/" + f"talk-bot-output/{filename}" + ) + + response = httpx.put( + f"{NC_URL}{dav_path}", + content=wav_bytes, + auth=(NC_USER, NC_PASSWORD), + timeout=120, + ) + + response.raise_for_status() + + share_path = f"/talk-bot-output/{filename}" + + log.info( + "Uploaded generated audio as %s", + share_path, + ) + + return share_path + + +def share_file_to_talk( + token: str, + file_path: str, + caption: str, +) -> None: + share_response = httpx.post( + f"{NC_URL}/ocs/v2.php/apps/files_sharing/api/v1/shares", + headers={ + "OCS-APIRequest": "true", + "Accept": "application/json", + }, + auth=(NC_USER, NC_PASSWORD), + data={ + "path": file_path, + "shareType": 10, + "shareWith": token, + "permissions": 1, + "talkMetaData": json.dumps({ + "messageType": "voice-message", + "caption": caption, + }), + }, + timeout=60, + ) + + log.info( + "Talk file share POST HTTP %s", + share_response.status_code, + ) + + if share_response.status_code not in (200, 201): + log.warning( + "Talk file share failed: %s", + share_response.text[:1000], + ) + share_response.raise_for_status() + + +def post_manager_message( + token: str, + message: str, +) -> int: + response = httpx.post( + f"{NC_URL}/ocs/v2.php/apps/spreed/api/v1/chat/{token}", + headers={ + "OCS-APIRequest": "true", + "Accept": "application/json", + }, + auth=(NC_USER, NC_PASSWORD), + data={ + "message": message, + }, + timeout=30, + ) + + log.info( + "Manager message POST HTTP %s", + response.status_code, + ) + + response.raise_for_status() + + try: + result = response.json() + + data = result.get("ocs", {}).get("data", {}) + + if isinstance(data, dict): + if "id" in data: + return int(data["id"]) + + if isinstance(data, list) and data: + if isinstance(data[0], dict) and "id" in data[0]: + return int(data[0]["id"]) + + except Exception: + pass + + return 0 + + +def make_filename( + sender_display_name: str, + target_language: str, +) -> str: + sender = sanitize_filename_part( + sender_display_name, + ) + + lang = language_short_code( + target_language, + ) + + now = datetime.now() + + return ( + f"{sender}-" + f"{lang}-" + f"{now:%Y%m%d}-" + f"{now:%H%M}.wav" + ) + + +def build_file_message( + file_path: str, + filename: str, + translated_text: str, +) -> str: + payload = { + "message": f"{translated_text}\n\n{{file}}", + "parameters": { + "file": { + "type": "file", + "name": filename, + "path": filename, + "mimetype": "audio/wav", + } + }, + } + + return json.dumps(payload) + + +def process_voice_message( + token: str, + message_id: int, + sender_user_id: str, + sender_display_name: str, + file_param: dict, + recipients: list[str], +) -> None: + file_id = file_param.get("id") + + if not file_id: + raise RuntimeError( + "Voice attachment does not contain a file id" + ) + + file_id = int(file_id) + + log.info( + "Voice message %s from %s in conversation %s", + message_id, + sender_user_id, + token, + ) + + log.info( + "File: id=%s name=%s mimetype=%s path=%s", + file_id, + file_param.get("name"), + file_param.get("mimetype"), + file_param.get("path"), + ) + + audio_bytes = download_attachment(file_id) + + text, detected_language = transcribe( + audio_bytes, + ) + + log.info( + "Transcribed (%s): %s", + detected_language, + text, + ) + + if is_blank_transcription(text): + log.info( + "Whisper returned blank audio for message %s; ignoring the rest", + message_id, + ) + return + + sender_language = normalize_language_code( + get_user_language(sender_user_id), + ) + + log.info( + "Sender %s language: %s -> %s", + sender_user_id, + sender_language, + language_name(sender_language), + ) + + if not recipients: + log.warning( + "No translation recipients found for conversation %s", + token, + ) + return + + translations = [] + + for recipient in recipients: + recipient_language = get_user_language( + recipient, + ) + + recipient_language = normalize_language_code( + recipient_language, + ) + + recipient_language_name = language_name( + recipient_language, + ) + + log.info( + "Recipient %s language: %s -> %s", + recipient, + recipient_language, + recipient_language_name, + ) + + if detected_language == recipient_language: + translated_text = text + else: + translated_text = translate( + text, + sender_language, + recipient_language, + ) + + log.info( + "Translated (%s): %s", + recipient_language_name, + translated_text, + ) + + translations.append( + ( + recipient, + recipient_language, + translated_text, + ) + ) + + unique_translations = [] + + seen_languages = set() + + for recipient, target_language, translated_text in translations: + if target_language in seen_languages: + continue + + seen_languages.add(target_language) + + unique_translations.append( + ( + recipient, + target_language, + translated_text, + ) + ) + + generated = [] + + for recipient, target_language, translated_text in unique_translations: + wav_bytes = synthesize( + translated_text, + target_language, + sender_user_id, + ) + + filename = make_filename( + sender_display_name, + target_language, + ) + + log.info( + "Generated filename: %s", + filename, + ) + + file_path = upload_audio( + wav_bytes, + filename, + ) + + generated.append( + ( + target_language, + filename, + file_path, + ) + ) + + if not generated: + return + + deleted = delete_message( + token, + message_id, + ) + + if not deleted: + log.warning( + "Original message %s was not deleted. " + "Generated translations will not be posted " + "to avoid confusing duplicates.", + message_id, + ) + return + + for target_language, filename, file_path in generated: + translated_text = next( + translated_text + for _, lang, translated_text in unique_translations + if lang == target_language + ) + + share_file_to_talk( + token, + file_path, + translated_text, + ) + + log.info( + "Posted translated audio %s (%s)", + filename, + language_name(target_language), + ) + + +@app.get("/health") +def health(): + return { + "status": "ok", + "manager": MANAGER_USER, + "llamacpp_timeout": LLAMACPP_TIMEOUT, + "llamacpp_thinking": LLAMACPP_ENABLE_THINKING, + "qwen_tts_timeout": QWEN_TTS_TIMEOUT, + } + + +@app.post("/webhook") +async def webhook( + request: Request, + x_nextcloud_talk_random: Optional[str] = Header(None), + x_nextcloud_talk_signature: Optional[str] = Header(None), +): + body = await request.body() + + if not verify_webhook_backend( + x_nextcloud_talk_random, + x_nextcloud_talk_signature, + body, + ): + raise HTTPException( + status_code=401, + detail="Invalid webhook signature", + ) + + try: + payload = json.loads(body) + except json.JSONDecodeError: + raise HTTPException( + status_code=400, + detail="Invalid JSON", + ) + + event_type = payload.get("type") + if event_type not in {"Create", "Activity"}: + return JSONResponse( + { + "status": "ignored", + "reason": f"unsupported event type: {event_type}", + } + ) + + actor = payload.get("actor", {}) + obj = payload.get("object", {}) + target = payload.get("target", {}) + + token = target.get("id") + message_id = obj.get("id") + + if not token or not message_id: + return JSONResponse( + { + "status": "ignored", + "reason": "missing token or message id", + } + ) + + sender_user_id = actor_user_id(actor) + sender_display_name = ( + actor.get("name") + or sender_user_id + or "User" + ) + + log.info( + "Incoming message %s from %s in conversation %s", + message_id, + sender_user_id, + token, + ) + + if sender_user_id == MANAGER_USER: + log.info( + "Ignoring Manager's own message %s", + message_id, + ) + return JSONResponse( + { + "status": "ignored", + "reason": "manager's own message", + } + ) + + content_raw = obj.get( + "content", + "{}", + ) + + try: + content = json.loads(content_raw) + except (json.JSONDecodeError, TypeError): + content = {} + + message_text = ( + content.get("message", "") + or "" + ).strip() + + parameters = content.get( + "parameters", + {}, + ) + + file_param = extract_file_param( + parameters, + ) + + try: + recipients = get_target_user_ids( + token, + sender_user_id, + ) + except Exception as exc: + log.exception( + "Could not load participants for conversation %s: %s", + token, + exc, + ) + recipients = [] + + if file_param: + try: + process_voice_message( + token=token, + message_id=int(message_id), + sender_user_id=sender_user_id, + sender_display_name=sender_display_name, + file_param=file_param, + recipients=recipients, + ) + + except Exception as exc: + log.exception( + "Voice processing failed for message %s: %s", + message_id, + exc, + ) + + return JSONResponse( + { + "status": "ok", + "type": "voice", + } + ) + + if message_text: + log.info( + "Text message %s from %s", + message_id, + sender_user_id, + ) + + for recipient in recipients: + target_language = normalize_language_code( + get_user_language(recipient), + ) + + translated = translate( + message_text, + "auto", + target_language, + ) + + if translated.strip().lower() == message_text.lower(): + continue + + post_manager_message( + token, + translated, + ) + + log.info( + "Posted text translation for %s: %s", + recipient, + target_language, + ) + + return JSONResponse( + { + "status": "ok", + "type": "text", + } + ) + + log.info( + "Message %s has no text or attachment", + message_id, + ) + + return JSONResponse( + { + "status": "ignored", + "reason": "empty message", + } + ) diff --git a/containers/talkbot/talkbot.Containerfile b/containers/talkbot/talkbot.Containerfile new file mode 100644 index 0000000..ebace17 --- /dev/null +++ b/containers/talkbot/talkbot.Containerfile @@ -0,0 +1,27 @@ +### Nextcloud Talk voice-translate bot +### Receives voice messages via the Talk bot webhook, transcribes them +### with whisper.cpp, translates with llama.cpp, and synthesizes the +### translation with qwen-tts, replying into the same conversation. +### +### BUILD: podman build -t talkbot:latest -f talkbot.Containerfile . +FROM python:3.12-slim + +USER root +EXPOSE 8100 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +RUN pip install --no-cache-dir fastapi uvicorn httpx + +COPY server.py /app/server.py +COPY entrypoint.sh /app/entrypoint.sh +RUN chmod +x /app/entrypoint.sh + +ENTRYPOINT ["/app/entrypoint.sh"] +CMD [] diff --git a/containers/talkbot/talkbot.container b/containers/talkbot/talkbot.container new file mode 100644 index 0000000..591b116 --- /dev/null +++ b/containers/talkbot/talkbot.container @@ -0,0 +1,45 @@ +[Unit] +Description=Nextcloud Talk voice-translate bot +After=network-online.target +Wants=network-online.target + +[Container] +Image=localhost/talkbot:latest +ContainerName=talkbot + +Network=internal.network +PublishPort=8100:8100 + +PodmanArgs=--group-add=keep-groups +SecurityLabelType=container_runtime_t + +Volume=/srv/containers/talkbot/voices:/app/voice-samples:Z,ro +Environment=VOICE_SAMPLES_DIR=/app/voice-samples + +# --- Nextcloud connection --- +Environment=NC_URL=https://your-nextcloud.example.tld +# Shared secret used when registering the bot via: +# occ talk:bot:install "TranslateBot" https://talkbot.example/webhook +Environment=NC_BOT_SECRET=changeme +# Nextcloud user + app password (Settings > Security > App passwords) used +# only to read user languages and download/upload chat attachments. +Environment=NC_USER=changeme +Environment=NC_PASSWORD=changeme + +# --- Backend services (adjust host:port to match your setup/network) --- +Environment=WHISPER_URL=http://whisper:8080 +Environment=LLAMACPP_URL=http://llm:7000 +#Environment=LLAMACPP_MODEL= +Environment=LLAMACPP_ENABLE_THINKING=false +Environment=QWEN_TTS_URL=http://qwen-tts:8000 + +# --- Behaviour --- +Environment=DEFAULT_TARGET_LANGUAGE=it +Environment=QWEN_TTS_SPEAKER=Ryan + +[Service] +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=default.target diff --git a/containers/whisper/whisper-cpu.Containerfile b/containers/whisper/whisper-cpu.Containerfile new file mode 100644 index 0000000..7b77b93 --- /dev/null +++ b/containers/whisper/whisper-cpu.Containerfile @@ -0,0 +1,40 @@ +### Whisper.cpp Container - CPU only +### High-performance Speech-to-Text using OpenAI's Whisper model +### Based on whisper.cpp: https://github.com/ggml-org/whisper.cpp +### +### BUILD: ./build-container-cpu.sh (compiles locally, CPU-only) +### THEN: podman build -t whisper:cpu-amd64 -f whisper-cpu.Containerfile . +### With custom model: podman build --build-arg MODELS="small" -t whisper:cpu-amd64 -f whisper-cpu.Containerfile . +FROM ubuntu:26.04 +ARG MODELS=small +USER root +EXPOSE 8080 +RUN apt-get update \ + && apt-get install -y curl ffmpeg nano \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /tmp/* /var/tmp/* \ + && rm -rf /var/lib/apt/lists/* \ + && find /var/cache/apt/archives /var/lib/apt/lists -not -name lock -type f -delete \ + && find /var/cache -type f -delete +WORKDIR /app +# Copy pre-compiled CPU-only binaries +COPY bin-cpu/ /app/ +RUN chmod +x /app/whisper-* +# Copy models downloader and entrypoint +COPY models/download-ggml-model.sh /app/ +COPY entrypoint.sh /app/ +RUN chmod +x /app/download-ggml-model.sh /app/entrypoint.sh +# Create models directory (will be mounted as volume at runtime) +RUN mkdir -p /app/models +# Set environment variables +ENV PATH=/app:$PATH +ENV LD_LIBRARY_PATH=/app:/usr/local/lib:/usr/lib:/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH +ENV HF_HUB_ENABLE_HF_TRANSFER=1 +# Whisper model configuration +# MODELS arg is passed but not used during build (downloaded at runtime) +ENV WHISPER_MODEL_NAME=${MODELS} +ENV WHISPER_MODEL_FILE=ggml-${MODELS}.bin +WORKDIR /app +ENTRYPOINT ["/app/entrypoint.sh"] +CMD []