feat(talkbot): add Chatterbox TTS backend, generalize TTS env vars

2026-09-12 23:38:46 +02:00
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+### Build the chatterbox image. USAGE: ./build-container.sh [cpu|rocm] (default: cpu)
+set -euo pipefail
+
+DEVICE="${1:-cpu}"
+
+case "$DEVICE" in
+    cpu|rocm) ;;
+    *)
+        echo "Usage: $0 [cpu|rocm]" >&2
+        exit 1
+        ;;
+esac
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+podman build \
+    --build-arg DEVICE="$DEVICE" \
+    -t "localhost/chatterbox:$DEVICE" \
+    -t localhost/chatterbox:latest \
+    -f "$SCRIPT_DIR/chatterbox.Containerfile" \
+    "$SCRIPT_DIR"
@@ -0,0 +1,42 @@
+### Chatterbox Multilingual TTS Container (https://huggingface.co/ResembleAI/chatterbox) — build with ./build-container.sh [cpu|rocm]
+ARG DEVICE=cpu
+ARG BASE_IMAGE_CPU=python:3.12-slim
+ARG BASE_IMAGE_ROCM=docker.io/rocm/pytorch-nightly
+
+FROM ${BASE_IMAGE_CPU} AS base-cpu
+FROM ${BASE_IMAGE_ROCM} AS base-rocm
+
+FROM base-${DEVICE}
+ARG DEVICE
+
+USER root
+EXPOSE 8000
+
+# sox/ffmpeg: needed at runtime for reference-audio format handling
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends sox ffmpeg curl \
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# CPU-only PyTorch (ROCm base image already ships its own build)
+RUN if [ "$DEVICE" = "cpu" ]; then \
+        pip install --no-cache-dir torch torchaudio --index-url https://download.pytorch.org/whl/cpu; \
+    fi
+
+# Install chatterbox-tts and a minimal API server
+RUN pip install --no-cache-dir chatterbox-tts fastapi uvicorn python-multipart soundfile
+
+# Models download at runtime here — mount as a volume to persist them
+RUN mkdir -p /app/models
+ENV HF_HOME=/app/models
+
+# Copy entrypoint / server script
+COPY entrypoint.sh /app/entrypoint.sh
+COPY server.py /app/server.py
+RUN chmod +x /app/entrypoint.sh
+
+ENTRYPOINT ["/app/entrypoint.sh"]
+CMD []
@@ -0,0 +1,13 @@
+chatterbox.example.com {
+	import gateway_error
+
+	request_body {
+		max_size 100MB
+	}
+
+	reverse_proxy chatterbox:8000
+
+	log {
+		output file /var/log/caddy/chatterbox_access.log
+	}
+}
@@ -0,0 +1,38 @@
+[Unit]
+Description=Chatterbox Multilingual TTS Server
+After=network-online.target
+Wants=network-online.target
+
+[Container]
+Image=localhost/chatterbox:latest
+ContainerName=chatterbox
+Network=internal.network
+
+# CPU-only container
+#Memory=8g
+#CPUs=4
+
+# ROCm
+AddDevice=/dev/kfd
+AddDevice=/dev/dri/renderD128
+PodmanArgs=--group-add=keep-groups --ipc=host --pids-limit=-1 --security-opt label=disable
+SecurityLabelType=container_runtime_t
+
+# HTTP API
+PublishPort=8002:8000
+
+# Persist downloaded model weights across restarts (adjust host path as needed)
+Volume=/srv/containers/chatterbox/models:/app/models:Z
+
+# Optional: tune voice generation (defaults: 0.5, 0.5)
+#Environment=CHATTERBOX_EXAGGERATION=0.5
+#Environment=CHATTERBOX_CFG_WEIGHT=0.5
+
+
+[Service]
+Restart=on-failure
+RestartSec=5
+TimeoutStartSec=300
+
+[Install]
+WantedBy=default.target
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Starts the FastAPI server; model weights download on first request into /app/models
+set -e
+
+# wait for GPU: torch.cuda.is_available() caches its first (possibly premature) result
+if [ -e /dev/kfd ]; then
+    echo "ROCm device detected, waiting for GPU to become available..."
+    for i in $(seq 1 15); do
+        if python3 -c "import sys, torch; sys.exit(0 if torch.cuda.is_available() else 1)" 2>/dev/null; then
+            echo "GPU is available."
+            break
+        fi
+        sleep 1
+    done
+fi
+
+echo "=== Chatterbox Server ==="
+echo
+
+exec uvicorn server:app --host 0.0.0.0 --port 8000 --app-dir /app "$@"
@@ -0,0 +1,115 @@
+"""Minimal FastAPI server exposing Chatterbox Multilingual TTS. https://huggingface.co/ResembleAI/chatterbox"""
+import io
+import os
+import tempfile
+
+import torch
+import torchaudio as ta
+from fastapi import FastAPI, File, Form, HTTPException, UploadFile
+from fastapi.responses import StreamingResponse
+from pydantic import BaseModel
+from chatterbox.mtl_tts import ChatterboxMultilingualTTS
+
+_forced_device = os.environ.get("CHATTERBOX_DEVICE", "").strip()
+if _forced_device:
+    DEVICE = _forced_device
+elif torch.cuda.is_available():  # ROCm builds expose the GPU via the CUDA API too
+    DEVICE = "cuda"
+else:
+    DEVICE = "cpu"
+
+EXAGGERATION = float(os.environ.get("CHATTERBOX_EXAGGERATION", "0.5"))
+CFG_WEIGHT = float(os.environ.get("CHATTERBOX_CFG_WEIGHT", "0.5"))
+
+# Full language name -> ISO code fallback, for callers that don't send language_code.
+_NAME_TO_CODE = {
+    "arabic": "ar", "danish": "da", "german": "de", "greek": "el", "english": "en",
+    "spanish": "es", "finnish": "fi", "french": "fr", "hebrew": "he", "hindi": "hi",
+    "italian": "it", "japanese": "ja", "korean": "ko", "malay": "ms", "dutch": "nl",
+    "norwegian": "no", "polish": "pl", "portuguese": "pt", "russian": "ru",
+    "swedish": "sv", "swahili": "sw", "turkish": "tr", "chinese": "zh",
+}
+
+
+def resolve_language_id(language: str, language_code: str) -> str:
+    code = (language_code or "").strip().lower()
+    if code:
+        return code
+    return _NAME_TO_CODE.get((language or "").strip().lower(), "en")
+
+
+app = FastAPI(title="Chatterbox server")
+
+model: ChatterboxMultilingualTTS | None = None
+
+
+@app.on_event("startup")
+def load_model():
+    global model
+    print(f"Loading Chatterbox Multilingual TTS on {DEVICE}...")
+    model = ChatterboxMultilingualTTS.from_pretrained(device=DEVICE)
+    print("Chatterbox model loaded.")
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok", "device": DEVICE}
+
+
+class SpeechRequest(BaseModel):
+    text: str
+    language: str = "English"
+    language_code: str = ""
+
+
+@app.post("/speech")
+def speech(req: SpeechRequest):
+    if model is None:
+        raise HTTPException(status_code=503, detail="Model not loaded yet")
+
+    wav = model.generate(
+        text=req.text,
+        language_id=resolve_language_id(req.language, req.language_code),
+        exaggeration=EXAGGERATION,
+        cfg_weight=CFG_WEIGHT,
+    )
+
+    buf = io.BytesIO()
+    ta.save(buf, wav, model.sr, format="wav")
+    buf.seek(0)
+    return StreamingResponse(buf, media_type="audio/wav")
+
+
+@app.post("/speech/clone")
+def speech_clone(
+    text: str = Form(...),
+    language: str = Form("English"),
+    language_code: str = Form(""),
+    ref_audio: UploadFile = File(...),
+):
+    """Clone a voice from ref_audio and synthesize text with it."""
+    if model is None:
+        raise HTTPException(status_code=503, detail="Model not loaded yet")
+
+    ref_bytes = ref_audio.file.read()
+
+    suffix = os.path.splitext(ref_audio.filename or "")[1] or ".wav"
+    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
+        tmp.write(ref_bytes)
+        tmp_path = tmp.name
+
+    try:
+        wav = model.generate(
+            text=text,
+            language_id=resolve_language_id(language, language_code),
+            audio_prompt_path=tmp_path,
+            exaggeration=EXAGGERATION,
+            cfg_weight=CFG_WEIGHT,
+        )
+    finally:
+        os.remove(tmp_path)
+
+    buf = io.BytesIO()
+    ta.save(buf, wav, model.sr, format="wav")
+    buf.seek(0)
+    return StreamingResponse(buf, media_type="audio/wav")
@@ -50,19 +50,25 @@ curl -X POST http://localhost:8080/inference \
 ```
 e salvando il campo `text` risultante in `sara.txt`.

+Il bot è agnostico rispetto al backend TTS: punta a un solo `TTS_URL` e
+sceglie tra `/speech` (voce preset, nessun campione utente) e
+`/speech/clone` (voce clonata) in base alla presenza del campione utente.
+Qualsiasi backend che esponga entrambi gli endpoint va bene (qwen-tts,
+omnivoice, chatterbox, ...); se il backend supporta solo `/speech/clone`
+(es. omnivoice), assicurati che ogni utente abbia sempre un campione
+vocale, altrimenti la richiesta di fallback a `/speech` fallirà.
+
 Il container qwen-tts carica **entrambi i modelli** (CustomVoice e Base)
 nello stesso processo (`QWEN_TTS_LOAD=customvoice,voiceclone`, default),
-così un'unica istanza espone sia `/speech` (voce preset) che
-`/speech/clone` (voce clonata): il bot punta a un solo `QWEN_TTS_URL` e
-sceglie l'endpoint giusto in base alla presenza del campione utente.
-Questo costa più RAM/tempo di avvio rispetto a un solo modello, ma evita
-di dover gestire due container qwen-tts separati.
+così un'unica istanza espone sia `/speech` che `/speech/clone`. Questo
+costa più RAM/tempo di avvio rispetto a un solo modello, ma evita di
+dover gestire due container qwen-tts separati.

 ## File

 | File                    | Scopo                                                              |
 | ------------------------ | ------------------------------------------------------------------ |
-| `server.py`               | Logica del bot: webhook, pipeline whisper→llama.cpp→qwen-tts, risposta |
+| `server.py`               | Logica del bot: webhook, pipeline whisper→llama.cpp→TTS, risposta |
 | `entrypoint.sh`            | Avvia uvicorn, valida le variabili d'ambiente richieste             |
 | `talkbot.Containerfile`   | Immagine del container                                             |
 | `talkbot.container`       | Quadlet Podman/systemd per l'esecuzione come servizio               |
@@ -102,7 +108,7 @@ Password per app), non la password reale dell'account.
 podman build -t talkbot:latest -f talkbot.Containerfile .
 cp talkbot.container ~/.config/containers/systemd/
 # modifica i valori NC_URL / NC_BOT_SECRET / NC_ADMIN_USER / NC_ADMIN_PASSWORD
-# e gli URL di whisper/llama.cpp/qwen-tts nel file .container
+# e gli URL di whisper/llama.cpp/TTS nel file .container
 systemctl --user daemon-reload
 systemctl --user start talkbot
 journalctl --user -u talkbot -f
@@ -131,11 +137,11 @@ conversazione Talk dove il bot è abilitato: vedi sezione "Stato dei test".
 | `LLAMACPP_MODEL`             | no           | (vuoto)              | Nome modello, se il tuo router llama.cpp lo richiede |
 | `LLAMACPP_API_KEY`           | no           | (vuoto)              | API key per llama.cpp (header `Authorization: Bearer ...`); se vuoto, nessun header è inviato |
 | `LLAMACPP_TIMEOUT`           | no           | `600`                | Timeout (secondi) per la chiamata a llama.cpp        |
-| `QWEN_TTS_URL`               | no           | `http://qwen-tts:8000` | Endpoint qwen-tts (deve girare con `QWEN_TTS_LOAD` includendo sia `customvoice` che `voiceclone`, default) |
-| `QWEN_TTS_TIMEOUT`           | no           | `1800`               | Timeout (secondi) per la chiamata a qwen-tts         |
+| `TTS_URL`                    | no           | `http://qwen-tts:8000` | Endpoint del backend TTS (qwen-tts, omnivoice, chatterbox, ...), deve esporre `/speech` e/o `/speech/clone` |
+| `TTS_TIMEOUT`                | no           | `1800`               | Timeout (secondi) per la chiamata al backend TTS      |
 | `VOICE_SAMPLES_DIR`           | no           | `/app/voice-samples`  | Cartella con i campioni `<talk-username>.wav` per il cloning |
 | `DEFAULT_TARGET_LANGUAGE`     | no           | `en`                 | Lingua di fallback se non si riesce a leggere quella dell'utente |
-| `QWEN_TTS_SPEAKER`            | no           | `Ryan`               | Voce preset usata quando non c'è un campione utente |
+| `TTS_SPEAKER`                 | no           | `Ryan`               | Voce preset usata quando non c'è un campione utente (solo backend con `/speech`) |
 | `QUEUE_DB_PATH`               | no           | `/app/data/queue.db` | File SQLite della coda dei messaggi in attesa (vedi sotto) |

 > **`NC_URL` e hostname interno**: se punti `NC_URL` all'hostname del
@@ -6,7 +6,7 @@ echo "=== Nextcloud Talk voice-translate bot ==="
 echo "NC_URL:        ${NC_URL:-<not set>}"
 echo "WHISPER_URL:   ${WHISPER_URL:-http://whisper:8080}"
 echo "LLAMACPP_URL:  ${LLAMACPP_URL:-http://llamacpp:7000}"
-echo "QWEN_TTS_URL:  ${QWEN_TTS_URL:-http://qwen-tts:8000}"
+echo "TTS_URL:       ${TTS_URL:-http://qwen-tts:8000}"
 echo "Default lang:  ${DEFAULT_TARGET_LANGUAGE:-en}"
 echo

@@ -39,16 +39,16 @@ LOG_WEBHOOK_PAYLOAD = (
    in {"1", "true", "yes", "on"}
 )
 LOG_TEXT_PREVIEW_CHARS = int(os.environ.get("LOG_TEXT_PREVIEW_CHARS") or "120")
-QWEN_TTS_URL = os.environ.get("QWEN_TTS_URL", "http://qwen-tts:8000").rstrip("/")
-QWEN_TTS_TIMEOUT = int(os.environ.get("QWEN_TTS_TIMEOUT") or "1800")
+TTS_URL = os.environ.get("TTS_URL", "http://qwen-tts:8000").rstrip("/")
+TTS_TIMEOUT = int(os.environ.get("TTS_TIMEOUT") or "1800")

 DEFAULT_TARGET_LANGUAGE = os.environ.get(
    "DEFAULT_TARGET_LANGUAGE",
    "it",
 ).strip()

-QWEN_TTS_SPEAKER = os.environ.get(
-    "QWEN_TTS_SPEAKER",
+TTS_SPEAKER = os.environ.get(
+    "TTS_SPEAKER",
    "Ryan",
 ).strip()

@@ -883,6 +883,7 @@ def synthesize(
        data = {
            "text": text,
            "language": target_name,
+            "language_code": language_short_code(target_language).lower(),
        }

        if ref_text:
@@ -892,7 +893,7 @@ def synthesize(

        with open(sample_path, "rb") as reference_audio:
            response = httpx.post(
-                f"{QWEN_TTS_URL}/speech/clone",
+                f"{TTS_URL}/speech/clone",
                data=data,
                files={
                    "ref_audio": (
@@ -901,24 +902,25 @@ def synthesize(
                        "audio/wav",
                    )
                },
-                timeout=QWEN_TTS_TIMEOUT,
+                timeout=TTS_TIMEOUT,
            )

    else:
        log.info(
            "No voice sample for user %s, using default speaker %s",
            sender_user_id,
-            QWEN_TTS_SPEAKER,
+            TTS_SPEAKER,
        )

        response = httpx.post(
-            f"{QWEN_TTS_URL}/speech",
+            f"{TTS_URL}/speech",
            json={
                "text": text,
                "language": target_name,
-                "speaker": QWEN_TTS_SPEAKER,
+                "language_code": language_short_code(target_language).lower(),
+                "speaker": TTS_SPEAKER,
            },
-            timeout=QWEN_TTS_TIMEOUT,
+            timeout=TTS_TIMEOUT,
        )

    response.raise_for_status()
@@ -1578,7 +1580,7 @@ def health():
        "whisper_timeout": WHISPER_TIMEOUT,
        "llamacpp_timeout": LLAMACPP_TIMEOUT,
        "llamacpp_thinking": LLAMACPP_ENABLE_THINKING,
-        "qwen_tts_timeout": QWEN_TTS_TIMEOUT,
+        "tts_timeout": TTS_TIMEOUT,
        "queue_size": queue_size(),
    }

@@ -43,10 +43,10 @@ Environment=LLAMACPP_MAX_TOKENS=8192
 Environment=LLAMACPP_ENABLE_THINKING=false
 Environment=LLAMACPP_TIMEOUT=1800

-# QWEN
-Environment=QWEN_TTS_URL=http://qwen-tts:8000
-Environment=QWEN_TTS_TIMEOUT=5400
-Environment=QWEN_TTS_SPEAKER=Ryan
+# TTS (qwen-tts, omnivoice, chatterbox, ... any backend exposing /speech + /speech/clone)
+Environment=TTS_URL=http://qwen-tts:8000
+Environment=TTS_TIMEOUT=5400
+Environment=TTS_SPEAKER=Ryan


 # --- Behaviour ---