diff --git a/containers/chatterbox/build-container.sh b/containers/tts/chatterbox/build-container.sh similarity index 100% rename from containers/chatterbox/build-container.sh rename to containers/tts/chatterbox/build-container.sh diff --git a/containers/chatterbox/chatterbox.Containerfile b/containers/tts/chatterbox/chatterbox.Containerfile similarity index 100% rename from containers/chatterbox/chatterbox.Containerfile rename to containers/tts/chatterbox/chatterbox.Containerfile diff --git a/containers/chatterbox/chatterbox.caddy b/containers/tts/chatterbox/chatterbox.caddy similarity index 100% rename from containers/chatterbox/chatterbox.caddy rename to containers/tts/chatterbox/chatterbox.caddy diff --git a/containers/chatterbox/chatterbox.container b/containers/tts/chatterbox/chatterbox.container similarity index 100% rename from containers/chatterbox/chatterbox.container rename to containers/tts/chatterbox/chatterbox.container diff --git a/containers/chatterbox/entrypoint.sh b/containers/tts/chatterbox/entrypoint.sh similarity index 100% rename from containers/chatterbox/entrypoint.sh rename to containers/tts/chatterbox/entrypoint.sh diff --git a/containers/chatterbox/server.py b/containers/tts/chatterbox/server.py similarity index 100% rename from containers/chatterbox/server.py rename to containers/tts/chatterbox/server.py diff --git a/containers/tts/mosstts/build-container.sh b/containers/tts/mosstts/build-container.sh new file mode 100755 index 0000000..290ace5 --- /dev/null +++ b/containers/tts/mosstts/build-container.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +### Build the mosstts image. USAGE: ./build-container.sh [cpu|rocm] (default: cpu) +set -euo pipefail + +DEVICE="${1:-cpu}" + +case "$DEVICE" in + cpu|rocm) ;; + *) + echo "Usage: $0 [cpu|rocm]" >&2 + exit 1 + ;; +esac + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +podman build \ + --build-arg DEVICE="$DEVICE" \ + -t "localhost/mosstts:$DEVICE" \ + -t localhost/mosstts:latest \ + -f "$SCRIPT_DIR/mosstts.Containerfile" \ + "$SCRIPT_DIR" diff --git a/containers/tts/mosstts/entrypoint.sh b/containers/tts/mosstts/entrypoint.sh new file mode 100755 index 0000000..7f54415 --- /dev/null +++ b/containers/tts/mosstts/entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Starts the FastAPI server; model weights download on first request into /app/models +set -e + +# wait for GPU: torch.cuda.is_available() caches its first (possibly premature) result +if [ -e /dev/kfd ]; then + echo "ROCm device detected, waiting for GPU to become available..." + for i in $(seq 1 15); do + if python3 -c "import sys, torch; sys.exit(0 if torch.cuda.is_available() else 1)" 2>/dev/null; then + echo "GPU is available." + break + fi + sleep 1 + done +fi + +echo "=== MOSS-TTS Server ===" +echo + +exec uvicorn server:app --host 0.0.0.0 --port 8000 --app-dir /app "$@" diff --git a/containers/tts/mosstts/mosstts.Containerfile b/containers/tts/mosstts/mosstts.Containerfile new file mode 100644 index 0000000..fd19670 --- /dev/null +++ b/containers/tts/mosstts/mosstts.Containerfile @@ -0,0 +1,47 @@ +### MOSS-TTS Container (https://github.com/OpenMOSS/MOSS-TTS) — build with ./build-container.sh [cpu|rocm] +ARG DEVICE=cpu +ARG BASE_IMAGE_CPU=python:3.12-slim +ARG BASE_IMAGE_ROCM=docker.io/rocm/pytorch-nightly + +FROM ${BASE_IMAGE_CPU} AS base-cpu +FROM ${BASE_IMAGE_ROCM} AS base-rocm + +FROM base-${DEVICE} +ARG DEVICE + +USER root +EXPOSE 8000 + +# ffmpeg/libsndfile: needed at runtime for reference-audio format handling +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg curl \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# CPU-only PyTorch (ROCm base image already ships its own torch/torchaudio build) +RUN if [ "$DEVICE" = "cpu" ]; then \ + pip install --no-cache-dir torch torchaudio --index-url https://download.pytorch.org/whl/cpu; \ + fi + +# Core MOSS-TTS runtime deps (from upstream pyproject.toml), installed directly +# instead of cloning the repo: AutoModel/AutoProcessor with trust_remote_code +# pull the actual modeling code from the Hugging Face model repo at runtime. +RUN pip install --no-cache-dir \ + safetensors numpy orjson tqdm PyYAML einops scipy librosa tiktoken \ + psutil packaging "transformers>=5.0.0" accelerate torchcodec \ + fastapi uvicorn python-multipart soundfile + +# Models download at runtime here — mount as a volume to persist them +RUN mkdir -p /app/models +ENV HF_HOME=/app/models + +# Copy entrypoint / server script +COPY entrypoint.sh /app/entrypoint.sh +COPY server.py /app/server.py +RUN chmod +x /app/entrypoint.sh + +ENTRYPOINT ["/app/entrypoint.sh"] +CMD [] diff --git a/containers/tts/mosstts/mosstts.caddy b/containers/tts/mosstts/mosstts.caddy new file mode 100644 index 0000000..f6c1338 --- /dev/null +++ b/containers/tts/mosstts/mosstts.caddy @@ -0,0 +1,13 @@ +mosstts.example.com { + import gateway_error + + request_body { + max_size 100MB + } + + reverse_proxy mosstts:8000 + + log { + output file /var/log/caddy/mosstts_access.log + } +} diff --git a/containers/tts/mosstts/mosstts.container b/containers/tts/mosstts/mosstts.container new file mode 100644 index 0000000..d92df3f --- /dev/null +++ b/containers/tts/mosstts/mosstts.container @@ -0,0 +1,46 @@ +[Unit] +Description=MOSS-TTS Server +After=network-online.target +Wants=network-online.target + +[Container] +Image=localhost/mosstts:latest +ContainerName=mosstts +Network=internal.network + +# CPU-only container +#Memory=8g +#CPUs=4 + +# ROCm +AddDevice=/dev/kfd +AddDevice=/dev/dri/renderD128 +PodmanArgs=--group-add=keep-groups --ipc=host --pids-limit=-1 --security-opt label=disable +SecurityLabelType=container_runtime_t + +# HTTP API +PublishPort=8003:8000 + +# Persist downloaded model weights across restarts (adjust host path as needed) +Volume=/srv/containers/mosstts/models:/app/models:Z + +# Default model: OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 (4B, 48kHz stereo). +# Other checkpoints usable via MOSSTTS_MODEL (same generate API unless noted): +# OpenMOSS-Team/MOSS-TTS-v1.5 8B, 24kHz, the flagship model +# OpenMOSS-Team/MOSS-TTSD-v1.0 8B, same arch as v1.5, earlier release +# OpenMOSS-Team/MOSS-TTS-Nano 0.1B, CPU-friendly, API not verified +# OpenMOSS-Team/MOSS-TTS-Realtime 1.7B, multi-turn streaming voice-agent +# model with a DIFFERENT generation API; +# NOT compatible with this server as-is +#Environment=MOSSTTS_MODEL=OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 +#Environment=MOSSTTS_ATTN_IMPL=sdpa +#Environment=MOSSTTS_MAX_NEW_TOKENS=4096 + + +[Service] +Restart=on-failure +RestartSec=5 +TimeoutStartSec=300 + +[Install] +WantedBy=default.target diff --git a/containers/tts/mosstts/server.py b/containers/tts/mosstts/server.py new file mode 100644 index 0000000..b9bc827 --- /dev/null +++ b/containers/tts/mosstts/server.py @@ -0,0 +1,149 @@ +"""Minimal FastAPI server exposing MOSS-TTS generation and voice cloning. https://github.com/OpenMOSS/MOSS-TTS + +Other checkpoints usable via MOSSTTS_MODEL (same build_user_message/generate/decode +API unless noted): +- OpenMOSS-Team/MOSS-TTS-v1.5 8B, 24kHz, the flagship model +- OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 4B, 48kHz stereo (default here) +- OpenMOSS-Team/MOSS-TTSD-v1.0 8B, same arch as v1.5, earlier release +- OpenMOSS-Team/MOSS-TTS-Nano 0.1B, CPU-friendly (as few as 4 cores), + streaming-oriented; API not verified +- OpenMOSS-Team/MOSS-TTS-Realtime 1.7B, multi-turn/streaming voice-agent + model with a different generation API + (not build_user_message/generate/decode); + NOT compatible with this server as-is +""" +import io +import os +import tempfile + +import soundfile as sf +import torch +from fastapi import FastAPI, File, Form, HTTPException, UploadFile +from fastapi.responses import StreamingResponse +from pydantic import BaseModel +from transformers import AutoModel, AutoProcessor + +MODEL_NAME = os.environ.get("MOSSTTS_MODEL", "OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5") +ATTN_IMPLEMENTATION = os.environ.get("MOSSTTS_ATTN_IMPL", "sdpa") +MAX_NEW_TOKENS = int(os.environ.get("MOSSTTS_MAX_NEW_TOKENS", "4096")) + +_forced_device = os.environ.get("MOSSTTS_DEVICE", "").strip() +if _forced_device: + DEVICE = _forced_device +elif torch.cuda.is_available(): # ROCm builds expose the GPU via the CUDA API too + DEVICE = "cuda" +else: + DEVICE = "cpu" + +# bfloat16 is not well supported on CPU; use full precision there. +DTYPE = torch.bfloat16 if DEVICE == "cuda" else torch.float32 + +if DEVICE == "cuda": + # Match the backends enabled in the upstream quickstart; avoids a broken + # cuDNN SDP kernel on some GPU combos (same class of issue as chatterbox). + torch.backends.cuda.enable_cudnn_sdp(False) + torch.backends.cuda.enable_flash_sdp(True) + torch.backends.cuda.enable_mem_efficient_sdp(True) + torch.backends.cuda.enable_math_sdp(True) + +app = FastAPI(title="MOSS-TTS server") + +model = None +processor = None + + +@app.on_event("startup") +def load_model(): + global model, processor + print(f"Loading MOSS-TTS processor/model {MODEL_NAME} on {DEVICE}...") + + processor = AutoProcessor.from_pretrained( + MODEL_NAME, + trust_remote_code=True, + ) + processor.audio_tokenizer = processor.audio_tokenizer.to(DEVICE) + + model = AutoModel.from_pretrained( + MODEL_NAME, + trust_remote_code=True, + attn_implementation=ATTN_IMPLEMENTATION, + dtype=DTYPE, + ).to(DEVICE) + model.eval() + + print("MOSS-TTS model loaded.") + + +@app.get("/health") +def health(): + return {"status": "ok", "model": MODEL_NAME, "device": DEVICE} + + +def _run_generation(text: str, language: str, reference: list[str] | None) -> tuple: + if model is None or processor is None: + raise HTTPException(status_code=503, detail="Model not loaded yet") + + message_kwargs = {"text": text} + if language: + message_kwargs["language"] = language + if reference: + message_kwargs["reference"] = reference + + conversation = [processor.build_user_message(**message_kwargs)] + batch = processor([conversation], mode="generation") + input_ids = batch["input_ids"].to(DEVICE) + attention_mask = batch["attention_mask"].to(DEVICE) + + with torch.no_grad(): + outputs = model.generate( + input_ids=input_ids, + attention_mask=attention_mask, + max_new_tokens=MAX_NEW_TOKENS, + ) + + decoded = processor.decode(outputs)[0] + audio = decoded.audio_codes_list[0] + if audio.ndim > 1: + audio = audio.squeeze(0) + + return audio.detach().cpu().to(torch.float32).numpy(), processor.model_config.sampling_rate + + +class SpeechRequest(BaseModel): + text: str + language: str = "" + + +@app.post("/speech") +def speech(req: SpeechRequest): + audio, sr = _run_generation(req.text, req.language, reference=None) + + buf = io.BytesIO() + sf.write(buf, audio, sr, format="WAV") + buf.seek(0) + return StreamingResponse(buf, media_type="audio/wav") + + +@app.post("/speech/clone") +def speech_clone( + text: str = Form(...), + language: str = Form(""), + ref_audio: UploadFile = File(...), +): + """Clone a voice from ref_audio and synthesize text with it.""" + ref_bytes = ref_audio.file.read() + + suffix = os.path.splitext(ref_audio.filename or "")[1] or ".wav" + with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp: + tmp.write(ref_bytes) + tmp_path = tmp.name + + try: + audio, sr = _run_generation(text, language, reference=[tmp_path]) + finally: + os.remove(tmp_path) + + buf = io.BytesIO() + sf.write(buf, audio, sr, format="WAV") + buf.seek(0) + return StreamingResponse(buf, media_type="audio/wav") diff --git a/containers/omnivoice/build-container.sh b/containers/tts/omnivoice/build-container.sh similarity index 100% rename from containers/omnivoice/build-container.sh rename to containers/tts/omnivoice/build-container.sh diff --git a/containers/omnivoice/entrypoint.sh b/containers/tts/omnivoice/entrypoint.sh similarity index 100% rename from containers/omnivoice/entrypoint.sh rename to containers/tts/omnivoice/entrypoint.sh diff --git a/containers/omnivoice/omnivoice.Containerfile b/containers/tts/omnivoice/omnivoice.Containerfile similarity index 100% rename from containers/omnivoice/omnivoice.Containerfile rename to containers/tts/omnivoice/omnivoice.Containerfile diff --git a/containers/omnivoice/omnivoice.caddy b/containers/tts/omnivoice/omnivoice.caddy similarity index 100% rename from containers/omnivoice/omnivoice.caddy rename to containers/tts/omnivoice/omnivoice.caddy diff --git a/containers/omnivoice/omnivoice.container b/containers/tts/omnivoice/omnivoice.container similarity index 100% rename from containers/omnivoice/omnivoice.container rename to containers/tts/omnivoice/omnivoice.container diff --git a/containers/omnivoice/server.py b/containers/tts/omnivoice/server.py similarity index 100% rename from containers/omnivoice/server.py rename to containers/tts/omnivoice/server.py diff --git a/containers/qwentts/build-container.sh b/containers/tts/qwentts/build-container.sh similarity index 100% rename from containers/qwentts/build-container.sh rename to containers/tts/qwentts/build-container.sh diff --git a/containers/qwentts/entrypoint.sh b/containers/tts/qwentts/entrypoint.sh similarity index 100% rename from containers/qwentts/entrypoint.sh rename to containers/tts/qwentts/entrypoint.sh diff --git a/containers/qwentts/qwentts.Containerfile b/containers/tts/qwentts/qwentts.Containerfile similarity index 100% rename from containers/qwentts/qwentts.Containerfile rename to containers/tts/qwentts/qwentts.Containerfile diff --git a/containers/qwentts/qwentts.caddy b/containers/tts/qwentts/qwentts.caddy similarity index 100% rename from containers/qwentts/qwentts.caddy rename to containers/tts/qwentts/qwentts.caddy diff --git a/containers/qwentts/qwentts.container b/containers/tts/qwentts/qwentts.container similarity index 100% rename from containers/qwentts/qwentts.container rename to containers/tts/qwentts/qwentts.container diff --git a/containers/qwentts/server.py b/containers/tts/qwentts/server.py similarity index 100% rename from containers/qwentts/server.py rename to containers/tts/qwentts/server.py