23 Zeilen
629 B
Bash
23 Zeilen
629 B
Bash
|
#!/bin/bash
|
||
|
|
|
||
|
|
# Prefer Q6 + mmap; fallback to no-mmap only if explicitly requested
|
||
|
|
EXTRA=""
|
||
|
|
if [[ "$FORCE_NO_MMAP_CODER" == "1" ]]; then EXTRA="--no-mmap"; fi
|
||
|
|
exec /app/llama-server $CODER_MEDIUM_MODEL \
|
||
|
|
-c $CODER_CONTEXT_SIZE -ngl $CODER_GPU_LAYERS -n $CODER_MAX_TOKENS \
|
||
|
|
--temp 0.5 --top-p 0.9 --top-k 40 --repeat-penalty 1.1 $EXTRA \
|
||
|
|
--flash-attn auto --threads -1 --threads-batch -1 --threads-http -1 \
|
||
|
|
--jinja \
|
||
|
|
--timeout 900 --host 0.0.0.0 --port 8095 &
|
||
|
|
PID=$!
|
||
|
|
|
||
|
|
cleanup() {
|
||
|
|
echo "Stopping llama-server..."
|
||
|
|
kill $PID 2>/dev/null
|
||
|
|
wait $PID 2>/dev/null
|
||
|
|
exit 0
|
||
|
|
}
|
||
|
|
|
||
|
|
trap cleanup SIGTERM SIGINT
|
||
|
|
|
||
|
|
wait $PID
|