Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env bash
- # Qwen3.8-27B (UD-Q4_K_XL) — llama-server @ 131,072 context
- # Target: Ryzen 5950X (16C/32T) + RTX 3060 12GB + 128GB DDR4
- #
- # ARCHITECTURE (from config.json, arch=qwen3_5):
- # 64 layers, hidden 5120, ffn 17408, dense (no experts)
- # HYBRID: full_attention_interval=4
- # -> 16 full-attention layers (every 4th: 3,7,11,...,63)
- # -> 48 linear-attention layers (recurrent, constant state, ctx-independent)
- # Full-attn: 24 Q heads / 4 KV heads, head_dim 256 (GQA 6:1)
- #
- # KV MATH (only the 16 full-attn layers scale with context):
- # 2 (K+V) x 4 heads x 256 dim x 16 layers = 64 KB/token at f16
- # 131072 @ f16 = 8.4 GB
- # 131072 @ q8_0 = 4.2 GB <-- this config
- # Halving ctx from 262144 freed 4.2 GB, which buys ~8 more layers on the GPU.
- # That is the whole point of this variant: fewer tokens, faster tokens.
- #
- # Weights 17.92 GB + KV 4.2 GB = ~22 GB across 12 GB VRAM + 128 GB RAM.
- set -euo pipefail
- MODEL_REPO="unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL"
- CTX=131072
- NGL=26 # of 64. Raise by 2 until OOM at load, then back off one.
- ./build/bin/llama-server \
- -hf "${MODEL_REPO}" \
- --host 0.0.0.0 \
- --port 9931 \
- --alias qwen3.8-27b \
- \
- `# ---- context & batching ----` \
- -c "${CTX}" \
- -np 1 \
- -b 2048 \
- -ub 512 \
- --cache-reuse 256 \
- \
- `# ---- placement ----` \
- -ngl "${NGL}" \
- --load-mode mlock \
- \
- `# ---- attention & KV ----` \
- -fa on \
- -ctk q8_0 \
- -ctv q8_0 \
- \
- `# ---- CPU threads (5950X = 16 physical cores) ----` \
- -t 16 \
- -tb 16 \
- --no-mmproj \
- `# ---- chat / reasoning (thinking mode is ON by default) ----` \
- --jinja \
- --chat-template-file qwen.jinja \
- --reasoning-format deepseek \
- --reasoning-preserve \
- \
- `# ---- sampling: Qwen THINKING-mode defaults ----` \
- --temp 1.0 \
- --top-p 0.95 \
- --top-k 20 \
- --min-p 0.0 \
- --presence-penalty 0.0 \
- --repeat-penalty 1.0 \
- \
- `# ---- telemetry ----` \
- --metrics \
- --log-timestamps \
- --spec-type draft-mtp \
- --spec-draft-n-max 2 \
- -ngld 99 \
- -ctkd q8_0 \
- -ctvd q8_0
Advertisement
Add Comment
Please, Sign In to add comment