LuffyTheFox

Untitled

Apr 5th, 2026 (edited)
162
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 18.54 KB | None | 0 0
  1. # ============================================================================
  2. # Unsloth UD-Q4_K_XL -> JSON profile -> delete ref -> quantize YOUR Q8_0 GGUF
  3. # ============================================================================
  4.  
  5. import os
  6. import sys
  7. import re
  8. import gc
  9. import json
  10. import time
  11. import subprocess
  12. import numpy as np
  13.  
  14. # ============================================================================
  15. # USER CONFIG
  16. # ============================================================================
  17.  
  18. WORK_DIR = "/content/gguf_workspace"
  19. os.makedirs(WORK_DIR, exist_ok=True)
  20.  
  21. # --- YOUR MODEL (input) ------------------------------------------------------
  22. # Put your trained model GGUF here (Q8_0 in your case):
  23. INPUT_FILE = os.path.join(WORK_DIR, "Qwen3.5-35B-A3B-Uncensored-KL.Q8_0.gguf")
  24.  
  25. # --- Unsloth reference quant (used only to extract profile) ------------------
  26. REF_FILE = os.path.join(WORK_DIR, "Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf")
  27. REF_URL = ("https://huggingface.co/unsloth/Qwen3.5-35B-A3B-GGUF/resolve/main/Qwen3.5-35B-A3B-UD-Q4_K_XL.gguf?download=true")
  28.  
  29. # --- Output ------------------------------------------------------------------
  30. TARGET_BASE_TYPE = "Q4_K_M"
  31. OUTPUT_TAG = "UD" # marker in output filename
  32.  
  33. # --- Store the extracted quant schema in JSON --------------------------------
  34. PROFILE_JSON = os.path.join(WORK_DIR, "unsloth_ud_profile.json")
  35.  
  36. # After JSON is built, delete Unsloth reference quant GGUF to save disk
  37. DELETE_REF_AFTER_PROFILE = True
  38.  
  39. # Keep JSON compact by default (do NOT store huge name->type map)
  40. # Turn on only if you really want the full audit dump (can be big).
  41. STORE_RAW_TENSOR_MAP = False
  42.  
  43. # --- llama.cpp binaries ------------------------------------------------------
  44. LLAMA_TAG = "b8192"
  45. LLAMA_URL = (
  46. "https://github.com/ggml-org/llama.cpp/releases/download/"
  47. f"{LLAMA_TAG}/llama-{LLAMA_TAG}-bin-ubuntu-x64.tar.gz"
  48. )
  49. LLAMA_TAR = f"llama-{LLAMA_TAG}-bin-ubuntu-x64.tar.gz"
  50. LLAMA_BIN_DIR = os.path.join(WORK_DIR, "llama-bin")
  51.  
  52. # Verbosity
  53. PRINT_PROFILE_SUMMARY = True
  54. VERIFY_AFTER_QUANT = True
  55.  
  56.  
  57. # ============================================================================
  58. # UTILS
  59. # ============================================================================
  60.  
  61. def ensure_python_pkg(pkg: str):
  62. subprocess.check_call(
  63. [sys.executable, "-m", "pip", "install", "-q", pkg],
  64. stdout=subprocess.DEVNULL,
  65. stderr=subprocess.DEVNULL,
  66. )
  67.  
  68. def file_size_gb(path):
  69. return os.path.getsize(path) / (1 << 30)
  70.  
  71. def disk_free_gb(path=WORK_DIR):
  72. st = os.statvfs(path)
  73. return (st.f_bavail * st.f_frsize) / (1 << 30)
  74.  
  75. def ram_used_gb():
  76. try:
  77. with open("/proc/self/status", "r") as f:
  78. for line in f:
  79. if line.startswith("VmRSS:"):
  80. return int(line.split()[1]) / (1 << 20)
  81. except Exception:
  82. pass
  83. return -1
  84.  
  85. def safe_remove(path):
  86. try:
  87. if os.path.isfile(path):
  88. os.remove(path)
  89. except Exception:
  90. pass
  91.  
  92. def download_file(url, out_path, label, timeout=6*3600):
  93. if os.path.isfile(out_path) and os.path.getsize(out_path) > 1024:
  94. print(f"[{label}] Found: {out_path} ({file_size_gb(out_path):.2f} GB)")
  95. return out_path
  96.  
  97. print(f"[{label}] Downloading -> {out_path}")
  98. os.makedirs(os.path.dirname(out_path), exist_ok=True)
  99.  
  100. for cmd in [
  101. ["wget", "-O", out_path, "-c", "--show-progress", url],
  102. ["curl", "-L", "-o", out_path, url],
  103. ]:
  104. try:
  105. subprocess.check_call(cmd, timeout=timeout)
  106. if os.path.isfile(out_path) and os.path.getsize(out_path) > 1024:
  107. print(f"[{label}] Downloaded: {out_path} ({file_size_gb(out_path):.2f} GB)")
  108. return out_path
  109. except Exception:
  110. continue
  111.  
  112. print(f"[{label}] ERROR: download failed")
  113. return None
  114.  
  115. def get_llama_quantize():
  116. # Try existing
  117. for root, _, files in os.walk(LLAMA_BIN_DIR):
  118. if "llama-quantize" in files:
  119. p = os.path.join(root, "llama-quantize")
  120. if os.access(p, os.X_OK):
  121. print(f"[BIN] Found: {p}")
  122. return p
  123.  
  124. # Download
  125. tar_path = os.path.join(WORK_DIR, LLAMA_TAR)
  126. ok = download_file(LLAMA_URL, tar_path, "BIN", timeout=1200)
  127. if not ok:
  128. return None
  129.  
  130. os.makedirs(LLAMA_BIN_DIR, exist_ok=True)
  131.  
  132. extracted = False
  133. for strip in [True, False]:
  134. try:
  135. cmd = ["tar", "xzf", tar_path, "-C", LLAMA_BIN_DIR]
  136. if strip:
  137. cmd.append("--strip-components=1")
  138. subprocess.check_call(cmd)
  139. extracted = True
  140. break
  141. except subprocess.CalledProcessError:
  142. continue
  143.  
  144. safe_remove(tar_path)
  145.  
  146. if not extracted:
  147. print("[BIN] ERROR: extraction failed.")
  148. return None
  149.  
  150. for root, _, files in os.walk(LLAMA_BIN_DIR):
  151. if "llama-quantize" in files:
  152. p = os.path.join(root, "llama-quantize")
  153. os.chmod(p, 0o755)
  154. print(f"[BIN] Ready: {p}")
  155. return p
  156.  
  157. print("[BIN] ERROR: llama-quantize not found.")
  158. return None
  159.  
  160. def normalize_output_basename(input_path):
  161. base = os.path.splitext(os.path.basename(input_path))[0]
  162. known_suffixes = [
  163. ".Q8_0", "-Q8_0", ".F16", "-F16", ".BF16", "-BF16", ".F32", "-F32",
  164. ".Q4_0", "-Q4_0", ".Q4_K_M", "-Q4_K_M", ".Q5_K_M", "-Q5_K_M",
  165. ".IQ4_XS", "-IQ4_XS", ".IQ4_NL", "-IQ4_NL",
  166. ]
  167. for suf in known_suffixes:
  168. if base.endswith(suf):
  169. base = base[:-len(suf)]
  170. break
  171. return base
  172.  
  173. def regex_escape_literal(s: str) -> str:
  174. return re.sub(r'([.^$*+?{}\[\]\\|()])', r'\\\1', s)
  175.  
  176. def normalize_qtype(s):
  177. if s is None:
  178. return None
  179. s = str(s).strip()
  180. s = s.replace("GGML_TYPE_", "").replace("ggml_type_", "").replace("GGUF_TYPE_", "")
  181. return s.upper()
  182.  
  183. def seems_quantized_from_types(type_list):
  184. # If any tensor type starts with Q or IQ, treat as already-quantized.
  185. for t in type_list:
  186. if not t:
  187. continue
  188. u = str(t).upper()
  189. if u.startswith("Q") or u.startswith("IQ") or u.startswith("TQ") or u.startswith("KQ"):
  190. return True
  191. return False
  192.  
  193. # ============================================================================
  194. # GGUF INSPECTION
  195. # ============================================================================
  196.  
  197. def read_gguf_tensors_basic(gguf_path):
  198. """
  199. Returns list of dicts: {name, type_str, n_bytes, data_offset}
  200. """
  201. ensure_python_pkg("gguf")
  202. from gguf import GGUFReader
  203.  
  204. r = GGUFReader(gguf_path)
  205. out = []
  206. for t in r.tensors:
  207. typ = None
  208. for a in ("tensor_type", "type", "ggml_type", "data_type"):
  209. if hasattr(t, a):
  210. typ = getattr(t, a)
  211. break
  212. typ_s = normalize_qtype(typ.name if hasattr(typ, "name") else typ)
  213.  
  214. n_bytes = getattr(t, "n_bytes", None)
  215. data_offset = getattr(t, "data_offset", None)
  216.  
  217. out.append({
  218. "name": t.name,
  219. "type": typ_s,
  220. "n_bytes": int(n_bytes) if n_bytes is not None else None,
  221. "data_offset": int(data_offset) if data_offset is not None else None,
  222. })
  223.  
  224. del r
  225. gc.collect()
  226. return out
  227.  
  228. def get_tensor_names(gguf_path):
  229. return [x["name"] for x in read_gguf_tensors_basic(gguf_path)]
  230.  
  231. def get_tensor_types_map(gguf_path):
  232. items = read_gguf_tensors_basic(gguf_path)
  233. return {x["name"]: x["type"] for x in items}
  234.  
  235. # ============================================================================
  236. # PROFILE EXTRACTION (from Unsloth ref GGUF)
  237. # ============================================================================
  238.  
  239. def extract_profile_from_ref(ref_path):
  240. """
  241. Build a compact rule-set that reproduces Unsloth per-tensor types.
  242.  
  243. Output JSON structure:
  244. {
  245. "ref_file": "...",
  246. "token_embd_type": "Q8_0",
  247. "output_type": "Q6_K",
  248. "rules": [
  249. {"pattern": "^blk\\.\\d+\\.attn_q\\.weight$", "type": "Q8_0", "kind": "blk_all_layers"},
  250. {"pattern": "^blk\\.(?:0|1|2)\\.ffn_down_exps\\.weight$", "type": "IQ4_NL", "kind": "blk_subset_layers"},
  251. ...
  252. ],
  253. "stats": {...}
  254. ["tensor_types_by_name": {...}] # optional
  255. }
  256. """
  257. types = get_tensor_types_map(ref_path)
  258.  
  259. token_embd_type = types.get("token_embd.weight")
  260. output_type = types.get("output.weight")
  261.  
  262. # group block tensors by (suffix,type)->layers
  263. blk_re = re.compile(r"^blk\.(\d+)\.(.+)$")
  264. by_suffix_type_layers = {} # (suffix, type) -> set(layers)
  265. globals_map = {} # name -> type
  266. max_layer = -1
  267.  
  268. for name, typ in types.items():
  269. m = blk_re.match(name)
  270. if not m:
  271. globals_map[name] = typ
  272. continue
  273. layer = int(m.group(1))
  274. suffix = m.group(2)
  275. max_layer = max(max_layer, layer)
  276. by_suffix_type_layers.setdefault((suffix, typ), set()).add(layer)
  277.  
  278. n_layers = max_layer + 1 if max_layer >= 0 else 0
  279.  
  280. rules = []
  281.  
  282. # Global exact rules (but skip token_embd/output since we’ll pass dedicated flags)
  283. for name, typ in sorted(globals_map.items()):
  284. if name in ("token_embd.weight", "output.weight"):
  285. continue
  286. if typ is None:
  287. continue
  288. rules.append({
  289. "pattern": "^" + regex_escape_literal(name) + "$",
  290. "type": typ,
  291. "kind": "global_exact",
  292. })
  293.  
  294. # Block rules
  295. for (suffix, typ), layers in sorted(by_suffix_type_layers.items(), key=lambda x: (x[0][0], str(x[0][1]))):
  296. if typ is None:
  297. continue
  298. layers = sorted(layers)
  299. suffix_esc = regex_escape_literal(suffix)
  300.  
  301. if n_layers and len(layers) == n_layers:
  302. pat = r"^blk\.\d+\." + suffix_esc + r"$"
  303. kind = "blk_all_layers"
  304. else:
  305. alt = "|".join(str(i) for i in layers)
  306. pat = r"^blk\.(?:" + alt + r")\." + suffix_esc + r"$"
  307. kind = "blk_subset_layers"
  308.  
  309. rules.append({
  310. "pattern": pat,
  311. "type": typ,
  312. "kind": kind,
  313. })
  314.  
  315. prof = {
  316. "ref_file": os.path.basename(ref_path),
  317. "created_unix": int(time.time()),
  318. "token_embd_type": token_embd_type,
  319. "output_type": output_type,
  320. "rules": rules,
  321. "stats": {
  322. "n_tensors_total": len(types),
  323. "n_layers_detected": n_layers,
  324. "n_rules": len(rules),
  325. },
  326. }
  327.  
  328. if STORE_RAW_TENSOR_MAP:
  329. prof["tensor_types_by_name"] = types
  330.  
  331. return prof
  332.  
  333. def load_or_build_profile_json():
  334. # If profile JSON exists, use it and do NOT download ref again.
  335. if os.path.isfile(PROFILE_JSON) and os.path.getsize(PROFILE_JSON) > 100:
  336. with open(PROFILE_JSON, "r", encoding="utf-8") as f:
  337. prof = json.load(f)
  338. print(f"[PROFILE] Loaded: {PROFILE_JSON}")
  339. return prof
  340.  
  341. print("[PROFILE] JSON not found; building from Unsloth reference GGUF...")
  342.  
  343. if not os.path.isfile(REF_FILE):
  344. download_file(REF_URL, REF_FILE, "REF", timeout=6*3600)
  345.  
  346. prof = extract_profile_from_ref(REF_FILE)
  347.  
  348. with open(PROFILE_JSON, "w", encoding="utf-8") as f:
  349. json.dump(prof, f, ensure_ascii=False, indent=2)
  350.  
  351. print(f"[PROFILE] Saved: {PROFILE_JSON}")
  352.  
  353. if DELETE_REF_AFTER_PROFILE:
  354. print(f"[PROFILE] Deleting Unsloth reference quant to save disk: {REF_FILE}")
  355. safe_remove(REF_FILE)
  356.  
  357. return prof
  358.  
  359. # ============================================================================
  360. # QUANTIZE
  361. # ============================================================================
  362.  
  363. def filter_rules_to_existing_tensors(rules, tensor_names):
  364. """
  365. Keep only rules whose regex matches at least one tensor name in your input model.
  366. This reduces command size and avoids useless rules.
  367. """
  368. kept = []
  369. names = tensor_names
  370.  
  371. for r in rules:
  372. pat = r.get("pattern")
  373. typ = r.get("type")
  374. if not pat or not typ:
  375. continue
  376. try:
  377. rx = re.compile(pat)
  378. except re.error:
  379. continue
  380. if any(rx.search(n) for n in names):
  381. kept.append(r)
  382.  
  383. return kept
  384.  
  385. def quantize_with_profile(input_path, output_path, profile):
  386. bin_path = get_llama_quantize()
  387. if not bin_path:
  388. raise RuntimeError("llama-quantize not available")
  389.  
  390. if not os.path.isfile(input_path):
  391. raise FileNotFoundError(f"Input not found: {input_path}")
  392.  
  393. # Disk check (quantization creates output + temp buffers)
  394. in_gb = file_size_gb(input_path)
  395. free_gb = disk_free_gb()
  396. need_gb = max(12.0, in_gb * 0.9)
  397. if free_gb < need_gb:
  398. raise RuntimeError(f"Not enough disk. Need ~{need_gb:.1f} GB free. Free={free_gb:.1f} GB")
  399.  
  400. # Decide if allow-requantize is needed (input already quantized)
  401. input_types_map = get_tensor_types_map(input_path)
  402. allow_requantize = seems_quantized_from_types(list(input_types_map.values()))
  403.  
  404. input_names = list(input_types_map.keys())
  405.  
  406. rules = profile.get("rules", [])
  407. rules = filter_rules_to_existing_tensors(rules, input_names)
  408.  
  409. env = os.environ.copy()
  410. bin_dir = os.path.dirname(bin_path)
  411. ld = env.get("LD_LIBRARY_PATH", "")
  412. env["LD_LIBRARY_PATH"] = f"{bin_dir}:{ld}" if ld else bin_dir
  413.  
  414. cmd = [bin_path]
  415.  
  416. if allow_requantize:
  417. cmd += ["--allow-requantize"]
  418.  
  419. # Apply dedicated types only if corresponding tensors exist in input
  420. if "token_embd.weight" in input_types_map and profile.get("token_embd_type"):
  421. cmd += ["--token-embedding-type", str(profile["token_embd_type"])]
  422. if "output.weight" in input_types_map and profile.get("output_type"):
  423. cmd += ["--output-tensor-type", str(profile["output_type"])]
  424.  
  425. for r in rules:
  426. cmd += ["--tensor-type", f"{r['pattern']}={r['type']}"]
  427.  
  428. cmd += [input_path, output_path, TARGET_BASE_TYPE]
  429.  
  430. print("\n" + "=" * 78)
  431. print("QUANTIZATION")
  432. print("=" * 78)
  433. print(f"[IN ] {input_path} ({in_gb:.2f} GB)")
  434. print(f"[OUT] {output_path}")
  435. print(f"[BASE] {TARGET_BASE_TYPE}")
  436. print(f"[RULES] {len(rules)} (filtered to your model)")
  437. print(f"[FLAG] allow_requantize = {allow_requantize}")
  438. print(f"[SYS] free disk = {free_gb:.1f} GB | ram used = {ram_used_gb():.2f} GB")
  439.  
  440. print("[CMD] (showing head)")
  441. print(" " + " \\\n ".join(cmd[:40]) + (" \\\n ... (many --tensor-type rules) ..." if len(cmd) > 40 else ""))
  442.  
  443. t0 = time.time()
  444. proc = subprocess.Popen(
  445. cmd,
  446. stdout=subprocess.PIPE,
  447. stderr=subprocess.STDOUT,
  448. text=True,
  449. env=env,
  450. bufsize=1,
  451. )
  452. for line in proc.stdout:
  453. line = line.rstrip()
  454. if line:
  455. print(" ", line)
  456. proc.wait()
  457.  
  458. if proc.returncode != 0:
  459. raise RuntimeError(f"llama-quantize failed (code={proc.returncode})")
  460.  
  461. dt = time.time() - t0
  462. out_gb = file_size_gb(output_path)
  463. print(f"[DONE] {dt/60:.1f} min | size {in_gb:.2f} GB -> {out_gb:.2f} GB")
  464.  
  465. def verify_output(output_path):
  466. print("[VERIFY] Checking output GGUF parse + quick zero-fill sample...")
  467. try:
  468. ensure_python_pkg("gguf")
  469. from gguf import GGUFReader
  470.  
  471. r = GGUFReader(output_path)
  472. tensors = list(r.tensors)
  473. print(f"[VERIFY] Tensors: {len(tensors)}")
  474.  
  475. checked = 0
  476. zeroish = 0
  477.  
  478. with open(output_path, "rb") as f:
  479. for t in tensors[:60]:
  480. n_bytes = getattr(t, "n_bytes", None)
  481. off = getattr(t, "data_offset", None)
  482. if n_bytes is None or off is None:
  483. continue
  484. n = min(4096, int(n_bytes))
  485. if n < 128:
  486. continue
  487. f.seek(int(off))
  488. raw = f.read(n)
  489. arr = np.frombuffer(raw, dtype=np.uint8)
  490. if arr.size and arr.sum() == 0:
  491. zeroish += 1
  492. checked += 1
  493.  
  494. del r
  495. gc.collect()
  496.  
  497. if zeroish:
  498. print(f"[VERIFY] WARNING: {zeroish}/{checked} sampled tensors looked zero-filled.")
  499. else:
  500. print(f"[VERIFY] OK: sampled {checked} tensors.")
  501. return True
  502. except Exception as e:
  503. print(f"[VERIFY] ERROR: {e}")
  504. return False
  505.  
  506.  
  507. # ============================================================================
  508. # MAIN
  509. # ============================================================================
  510.  
  511. def main():
  512. print("=" * 78)
  513. print("UNSLOTH UD-IQ4_XS PROFILE -> JSON -> DELETE REF -> QUANTIZE YOUR MODEL")
  514. print("=" * 78)
  515. print(f"[INFO] Work dir: {WORK_DIR}")
  516. print(f"[INFO] Input: {INPUT_FILE}")
  517. print(f"[INFO] Profile: {PROFILE_JSON}")
  518. print(f"[INFO] Target: {TARGET_BASE_TYPE}")
  519. print(f"[INFO] Disk free: {disk_free_gb():.2f} GB | RAM used: {ram_used_gb():.2f} GB")
  520.  
  521. if not os.path.isfile(INPUT_FILE):
  522. sys.exit(f"[ERROR] Your input GGUF not found: {INPUT_FILE}")
  523.  
  524. # Build/load JSON profile (this step deletes REF_FILE if it had to download it)
  525. profile = load_or_build_profile_json()
  526.  
  527. if PRINT_PROFILE_SUMMARY:
  528. print("[PROFILE] Summary:")
  529. print(json.dumps(profile.get("stats", {}), indent=2, ensure_ascii=False))
  530. print(f"[PROFILE] token_embd_type: {profile.get('token_embd_type')}")
  531. print(f"[PROFILE] output_type: {profile.get('output_type')}")
  532. print(f"[PROFILE] rules_total: {len(profile.get('rules', []))}")
  533.  
  534. base = normalize_output_basename(INPUT_FILE)
  535. output_path = os.path.join(WORK_DIR, f"{base}-{OUTPUT_TAG}-{TARGET_BASE_TYPE}.gguf")
  536.  
  537. quantize_with_profile(INPUT_FILE, output_path, profile)
  538.  
  539. if VERIFY_AFTER_QUANT:
  540. print("\n" + "=" * 78)
  541. verify_output(output_path)
  542.  
  543. print("\n" + "=" * 78)
  544. print("COMPLETE")
  545. print("=" * 78)
  546. print(f"[OUT] {output_path}")
  547. print(f"[OUT] Size: {file_size_gb(output_path):.2f} GB")
  548. print(f"[SYS] Free disk: {disk_free_gb():.2f} GB")
  549.  
  550. print("\n[FS]")
  551. for fn in sorted(os.listdir(WORK_DIR)):
  552. fp = os.path.join(WORK_DIR, fn)
  553. if os.path.isfile(fp):
  554. try:
  555. print(f" {fn} ({file_size_gb(fp):.2f} GB)")
  556. except Exception:
  557. print(f" {fn}")
  558.  
  559. if __name__ == "__main__":
  560. main()
Advertisement
Add Comment
Please, Sign In to add comment