32 línte
838 B
Bash
Comhad Infheidhmithe
32 línte
838 B
Bash
Comhad Infheidhmithe
#!/bin/bash
|
|
|
|
# Large model: auto-select mmap based on GPU layers
|
|
# <= 45 layers: use mmap (less VRAM usage, faster startup)
|
|
# > 45 layers: disable mmap (avoids SVM limits)
|
|
LAYERS=${CODER_TOP_GPU_LAYERS:-55}
|
|
MMAP_OPT=""
|
|
if [ "$LAYERS" -gt 45 ]; then
|
|
MMAP_OPT="--no-mmap"
|
|
echo "Using --no-mmap (layers=$LAYERS > 45)"
|
|
else
|
|
echo "Using mmap (layers=$LAYERS <= 45)"
|
|
fi
|
|
exec /app/llama-server $CODER_TOP_MODEL \
|
|
-c $CODER_CONTEXT_SIZE -ngl $LAYERS -n $CODER_MAX_TOKENS \
|
|
--temp 0.45 --top-p 0.9 --top-k 40 --repeat-penalty 1.12 \
|
|
$MMAP_OPT \
|
|
--flash-attn auto --threads -1 --threads-batch -1 --threads-http -1 \
|
|
--jinja \
|
|
--timeout 1200 --host 0.0.0.0 --port 8096 &
|
|
PID=$!
|
|
|
|
cleanup() {
|
|
echo "Stopping llama-server..."
|
|
kill $PID 2>/dev/null
|
|
wait $PID 2>/dev/null
|
|
exit 0
|
|
}
|
|
|
|
trap cleanup SIGTERM SIGINT
|
|
|
|
wait $PID |