Guest User

Untitled

a guest
Aug 14th, 2026
502
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 2.08 KB | None | 0 0
  1. #!/usr/bin/env bash
  2. # Qwen3.8-27B (UD-Q4_K_XL) — llama-server @ 131,072 context
  3. # Target: Ryzen 5950X (16C/32T) + RTX 3060 12GB + 128GB DDR4
  4. #
  5. # ARCHITECTURE (from config.json, arch=qwen3_5):
  6. #   64 layers, hidden 5120, ffn 17408, dense (no experts)
  7. #   HYBRID: full_attention_interval=4
  8. #     -> 16 full-attention layers (every 4th: 3,7,11,...,63)
  9. #     -> 48 linear-attention layers (recurrent, constant state, ctx-independent)
  10. #   Full-attn: 24 Q heads / 4 KV heads, head_dim 256 (GQA 6:1)
  11. #
  12. # KV MATH (only the 16 full-attn layers scale with context):
  13. #   2 (K+V) x 4 heads x 256 dim x 16 layers = 64 KB/token at f16
  14. #     131072 @ f16  = 8.4 GB
  15. #     131072 @ q8_0 = 4.2 GB   <-- this config
  16. #   Halving ctx from 262144 freed 4.2 GB, which buys ~8 more layers on the GPU.
  17. #   That is the whole point of this variant: fewer tokens, faster tokens.
  18. #
  19. # Weights 17.92 GB + KV 4.2 GB = ~22 GB across 12 GB VRAM + 128 GB RAM.
  20.  
  21. set -euo pipefail
  22.  
  23. MODEL_REPO="unsloth/Qwen3.8-27B-GGUF:UD-Q4_K_XL"
  24. CTX=131072
  25. NGL=26          # of 64. Raise by 2 until OOM at load, then back off one.
  26.  
  27. ./build/bin/llama-server \
  28.   -hf "${MODEL_REPO}" \
  29.   --host 0.0.0.0 \
  30.   --port 9931 \
  31.   --alias qwen3.8-27b \
  32.   \
  33.   `# ---- context & batching ----` \
  34.   -c "${CTX}" \
  35.   -np 1 \
  36.   -b 2048 \
  37.   -ub 512 \
  38.   --cache-reuse 256 \
  39.   \
  40.   `# ---- placement ----` \
  41.   -ngl "${NGL}" \
  42.   --load-mode mlock \
  43.   \
  44.   `# ---- attention & KV ----` \
  45.   -fa on \
  46.   -ctk q8_0 \
  47.   -ctv q8_0 \
  48.   \
  49.   `# ---- CPU threads (5950X = 16 physical cores) ----` \
  50.   -t 16 \
  51.   -tb 16 \
  52.   --no-mmproj \
  53.   `# ---- chat / reasoning (thinking mode is ON by default) ----` \
  54.   --jinja \
  55.   --chat-template-file qwen.jinja \
  56.   --reasoning-format deepseek \
  57.   --reasoning-preserve \
  58.   \
  59.   `# ---- sampling: Qwen THINKING-mode defaults ----` \
  60.   --temp 1.0 \
  61.   --top-p 0.95 \
  62.   --top-k 20 \
  63.   --min-p 0.0 \
  64.   --presence-penalty 0.0 \
  65.   --repeat-penalty 1.0 \
  66.   \
  67.   `# ---- telemetry ----` \
  68.   --metrics \
  69.   --log-timestamps \
  70.   --spec-type draft-mtp \
  71.   --spec-draft-n-max 2 \
  72.   -ngld 99 \
  73.   -ctkd q8_0 \
  74.   -ctvd q8_0
Advertisement
Add Comment
Please, Sign In to add comment