diff --git a/containers/talkbot/server.py b/containers/talkbot/server.py index 2924743..08e4b79 100644 --- a/containers/talkbot/server.py +++ b/containers/talkbot/server.py @@ -27,6 +27,7 @@ WHISPER_URL = os.environ.get("WHISPER_URL", "http://whisper:8080").rstrip("/") LLAMACPP_URL = os.environ.get("LLAMACPP_URL", "http://llamacpp:7000").rstrip("/") LLAMACPP_MODEL = os.environ.get("LLAMACPP_MODEL", "").strip() LLAMACPP_TIMEOUT = int(os.environ.get("LLAMACPP_TIMEOUT") or "600") +LLAMACPP_MAX_TOKENS = int(os.environ.get("LLAMACPP_MAX_TOKENS") or "8192") LLAMACPP_ENABLE_THINKING = ( os.environ.get("LLAMACPP_ENABLE_THINKING", "false").lower() in {"1", "true", "yes", "on"} @@ -643,6 +644,7 @@ def translate( }, ], "temperature": 0.1, + "max_tokens": LLAMACPP_MAX_TOKENS, "chat_template_kwargs": { "enable_thinking": LLAMACPP_ENABLE_THINKING, }, diff --git a/containers/talkbot/talkbot.container b/containers/talkbot/talkbot.container index 3e07d45..38e1a65 100644 --- a/containers/talkbot/talkbot.container +++ b/containers/talkbot/talkbot.container @@ -35,6 +35,7 @@ Environment=NC_PASSWORD=changeme Environment=WHISPER_URL=http://whisper:8080 Environment=LLAMACPP_URL=http://llamacpp:8090 #Environment=LLAMACPP_MODEL= +Environment=LLAMACPP_MAX_TOKENS=8192 Environment=LLAMACPP_ENABLE_THINKING=false Environment=QWEN_TTS_URL=http://qwen-tts:8000