first commit

Services/llamacpp/Scripts/startCoderTop.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Large model: auto-select mmap based on GPU layers
+# <= 45 layers: use mmap (less VRAM usage, faster startup)
+# > 45 layers: disable mmap (avoids SVM limits)
+LAYERS=${CODER_TOP_GPU_LAYERS:-55}
+MMAP_OPT=""
+if [ "$LAYERS" -gt 45 ]; then
+  MMAP_OPT="--no-mmap"
+  echo "Using --no-mmap (layers=$LAYERS > 45)"
+else
+  echo "Using mmap (layers=$LAYERS <= 45)"
+fi
+exec /app/llama-server $CODER_TOP_MODEL \
+  -c $CODER_CONTEXT_SIZE -ngl $LAYERS -n $CODER_MAX_TOKENS \
+  --temp 0.45 --top-p 0.9 --top-k 40 --repeat-penalty 1.12 \
+  $MMAP_OPT \
+  --flash-attn auto --threads -1 --threads-batch -1 --threads-http -1 \
+  --jinja \
+  --timeout 1200 --host 0.0.0.0 --port 8096 &
+PID=$!
+
+cleanup() {
+    echo "Stopping llama-server..."
+    kill $PID 2>/dev/null
+    wait $PID 2>/dev/null
+    exit 0
+}
+
+trap cleanup SIGTERM SIGINT
+
+wait $PID