LuffyTheFox

Untitled

Mar 25th, 2026 (edited)
149
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 25.99 KB | None | 0 0
  1. # ============================================================================
  2. # GGUF Pipeline: Q4_K_M → float32 merge → Q4_0
  3. #
  4. # Disk-efficient sequential download:
  5. # Peak disk ≈ 3 × model_size (not 4×)
  6. #
  7. # Flow:
  8. # 0) Download A (Q4_K_M), convert → Q4_0 via llama-quantize, delete A
  9. # 1) Download B, C (Q4_K_M)
  10. # 2) Cross-quant merge in float32: A(Q4_0) + (B−C)(Q4_K_M) → write Q4_0
  11. # 3) Delete B, C
  12. # 4) (optional) llama-quantize Q4_0 → Q4_K_M (single clean quantization disabled by default)
  13. #
  14. # Only ONE quantization step touch the final data:
  15. # - merge result → Q4_0 (intermediate)
  16. # ============================================================================
  17.  
  18. import subprocess, sys, os, gc, time, shutil
  19. from math import gcd
  20. import numpy as np
  21.  
  22. WORK_DIR = "/content/gguf_workspace"
  23. os.makedirs(WORK_DIR, exist_ok=True)
  24.  
  25. def install():
  26. for pkg in ["huggingface_hub[hf_xfer]", "gguf"]:
  27. subprocess.check_call(
  28. [sys.executable, "-m", "pip", "install", "-q", pkg],
  29. stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
  30. )
  31. print("[OK] Dependencies installed.")
  32.  
  33. install()
  34.  
  35. from huggingface_hub import hf_hub_download
  36. from gguf import GGUFReader
  37. from gguf.constants import GGMLQuantizationType, GGML_QUANT_SIZES
  38. from gguf import quants as gguf_quants
  39.  
  40. # ============================================================================
  41. # PATCH IQ block sizes
  42. # ============================================================================
  43.  
  44. _IQ = {
  45. "IQ1_S":(256,50),"IQ1_M":(256,56),"IQ2_XXS":(256,66),"IQ2_XS":(256,74),
  46. "IQ2_S":(256,82),"IQ3_XXS":(256,98),"IQ3_S":(256,110),
  47. "IQ4_NL":(32,18),"IQ4_XS":(256,136),
  48. }
  49. for _n, _g in _IQ.items():
  50. _qt = getattr(GGMLQuantizationType, _n, None)
  51. if _qt is not None and _qt not in GGML_QUANT_SIZES:
  52. GGML_QUANT_SIZES[_qt] = _g
  53.  
  54. # ============================================================================
  55. # PROBE quantize support
  56. # ============================================================================
  57.  
  58. _QUANTIZE_SUPPORTED = set()
  59.  
  60. def _probe_quantize():
  61. for qt in GGMLQuantizationType:
  62. if qt in (GGMLQuantizationType.F32, GGMLQuantizationType.F16,
  63. GGMLQuantizationType.BF16):
  64. _QUANTIZE_SUPPORTED.add(qt)
  65. elif qt in GGML_QUANT_SIZES:
  66. be, _ = GGML_QUANT_SIZES[qt]
  67. try:
  68. r = gguf_quants.quantize(np.zeros(be, np.float32), qt)
  69. if r is not None and len(r.ravel().tobytes()) > 0:
  70. _QUANTIZE_SUPPORTED.add(qt)
  71. except Exception:
  72. pass
  73.  
  74. _probe_quantize()
  75. print(f"[INFO] quantize() supported: {sorted([q.name for q in _QUANTIZE_SUPPORTED])}")
  76.  
  77. # ============================================================================
  78. # CONFIGURATION
  79. # ============================================================================
  80.  
  81. MODEL_A = {
  82. "repo": "HauhauCS/Qwen3.5-35B-A3B-Uncensored-HauhauCS-Aggressive",
  83. "file": "Qwen3.5-35B-A3B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf",
  84. }
  85. MODEL_B = {
  86. "repo": "samuelcardillo/Qwen3.5-35B-A3B-Claude-4.6-Opus-Reasoning-Distilled-GGUF",
  87. "file": "Qwen3.5-35B-A3B-Claude-4.6-Opus-Reasoning-Distilled-Q4_K_M.gguf",
  88. }
  89. MODEL_C = {
  90. "repo": "unsloth/Qwen3.5-35B-A3B-GGUF",
  91. "file": "Qwen3.5-35B-A3B-Q4_K_M.gguf",
  92. }
  93.  
  94. MERGE_WEIGHT = 1.0
  95. MERGED_NAME = "Qwen3.5-35B-Q4_0-merged.gguf"
  96. MERGED_PATH = os.path.join(WORK_DIR, MERGED_NAME)
  97.  
  98. MERGE_CHUNK_ELEMS = 2 * 1024 * 1024
  99.  
  100. # --- Final output (set None to "Q4_K_M" if you want to quantize final model) ---
  101. FINAL_QTYPE = None
  102.  
  103. # llama.cpp Vulkan build
  104. LLAMA_URL = "https://github.com/ggml-org/llama.cpp/releases/download/b8468/llama-b8468-bin-ubuntu-vulkan-x64.tar.gz"
  105. LLAMA_TAR = "llama-b8468-bin-ubuntu-vulkan-x64.tar.gz"
  106.  
  107. # ============================================================================
  108. # CONSTANTS
  109. # ============================================================================
  110.  
  111. FLOAT_QTYPES = {
  112. GGMLQuantizationType.F32,
  113. GGMLQuantizationType.F16,
  114. GGMLQuantizationType.BF16,
  115. }
  116.  
  117. FINETUNE_SUFFIXES = (
  118. "attn_q.weight","attn_k.weight","attn_v.weight","attn_output.weight",
  119. "ssm_in.weight","ssm_out.weight","ssm_a.weight",
  120. "ssm_d.weight","ssm_dt.weight",
  121. "ffn_gate.weight","ffn_up.weight","ffn_down.weight",
  122. "ffn_gate_exps.weight","ffn_up_exps.weight","ffn_down_exps.weight",
  123. "ffn_gate_inp.weight","ffn_gate_shexp.weight",
  124. "ffn_up_shexp.weight","ffn_down_shexp.weight",
  125. "attn_gate.weight",
  126. )
  127.  
  128. PROTECTED_SUBSTRINGS = (
  129. "token_embd","output.weight",
  130. "_norm",".bias",
  131. "rope_freqs","pos_embd",
  132. "v.blk.","v.patch_embd","v.position_embd",
  133. "v.post_ln","v.pre_ln","v.output_norm","v.proj",
  134. "mm.","mtp.",
  135. )
  136.  
  137. # ============================================================================
  138. # QUANT GEOMETRY
  139. # ============================================================================
  140.  
  141. def quant_block_elements(qt):
  142. if qt in FLOAT_QTYPES: return 1
  143. if qt in GGML_QUANT_SIZES: return GGML_QUANT_SIZES[qt][0]
  144. return 1
  145.  
  146. def quant_block_bytes(qt):
  147. if qt == GGMLQuantizationType.F32: return 4
  148. if qt == GGMLQuantizationType.F16: return 2
  149. if qt == GGMLQuantizationType.BF16: return 2
  150. if qt in GGML_QUANT_SIZES: return GGML_QUANT_SIZES[qt][1]
  151. return 1
  152.  
  153. def elems_to_raw(n_elems, qt):
  154. be = quant_block_elements(qt)
  155. bb = quant_block_bytes(qt)
  156. return (n_elems // be) * bb
  157.  
  158. # ============================================================================
  159. # UTILITY
  160. # ============================================================================
  161.  
  162. def disk_free_gb(p=WORK_DIR):
  163. st = os.statvfs(p); return (st.f_bavail * st.f_frsize) / 1073741824
  164.  
  165. def file_size_gb(p):
  166. return os.path.getsize(p) / 1073741824
  167.  
  168. def fmt_bytes(n):
  169. for u in ("B","KB","MB","GB"):
  170. if n < 1024: return f"{n:.1f} {u}"
  171. n /= 1024
  172. return f"{n:.1f} TB"
  173.  
  174. def ram_used_gb():
  175. try:
  176. with open("/proc/self/status") as f:
  177. for line in f:
  178. if line.startswith("VmRSS:"):
  179. return int(line.split()[1]) / 1048576
  180. except Exception:
  181. pass
  182. return -1
  183.  
  184. def safe_remove(path, protect=None):
  185. try:
  186. a = os.path.abspath(path)
  187. if protect and a == os.path.abspath(protect): return
  188. if os.path.isfile(a):
  189. sz = file_size_gb(a)
  190. os.remove(a)
  191. print(f"[CLEANUP] Removed {os.path.basename(a)} ({sz:.2f} GB)")
  192. except OSError:
  193. pass
  194.  
  195. def flush_and_close(f, label="file"):
  196. f.flush(); os.fsync(f.fileno()); f.close()
  197. print(f"[FLUSH] {label}: flush + fsync done.")
  198.  
  199. def align_down(v, a):
  200. if a <= 0: return v
  201. return (v // a) * a
  202.  
  203. def check_gpu():
  204. print("\n[GPU] Checking GPU...")
  205. try:
  206. result = subprocess.run(
  207. ["nvidia-smi", "--query-gpu=name,memory.total,driver_version",
  208. "--format=csv,noheader"],
  209. capture_output=True, text=True, timeout=10,
  210. )
  211. if result.returncode == 0:
  212. for line in result.stdout.strip().split('\n'):
  213. print(f"[GPU] {line.strip()}")
  214. return True
  215. except Exception:
  216. pass
  217. print("[GPU] nvidia-smi not available.")
  218. return False
  219.  
  220. # ============================================================================
  221. # GGUF STRUCTURE
  222. # ============================================================================
  223.  
  224. def parse_gguf_structure(path):
  225. reader = GGUFReader(path)
  226. tensors = {}
  227. for t in reader.tensors:
  228. tensors[t.name] = {
  229. "offset": int(t.data_offset),
  230. "size": int(t.n_bytes),
  231. "qtype": t.tensor_type,
  232. "qtype_name": t.tensor_type.name,
  233. "shape": tuple(t.shape),
  234. "n_elements": int(t.n_elements),
  235. }
  236. n = len(tensors)
  237. del reader; gc.collect()
  238. return tensors, n
  239.  
  240. # ============================================================================
  241. # RAW FILE I/O
  242. # ============================================================================
  243.  
  244. def read_chunk(f, tinfo, offset, size):
  245. actual = min(size, tinfo["size"] - offset)
  246. if actual <= 0: return b""
  247. f.seek(tinfo["offset"] + offset)
  248. return f.read(actual)
  249.  
  250. def write_chunk(f, tinfo, offset, data):
  251. f.seek(tinfo["offset"] + offset)
  252. f.write(data)
  253.  
  254. # ============================================================================
  255. # DEQUANTIZE / QUANTIZE
  256. # ============================================================================
  257.  
  258. def bytes_to_f32(raw, qt):
  259. if qt == GGMLQuantizationType.F32:
  260. return np.frombuffer(raw, dtype=np.float32).copy()
  261. if qt == GGMLQuantizationType.F16:
  262. return np.frombuffer(raw, dtype=np.float16).astype(np.float32)
  263. if qt == GGMLQuantizationType.BF16:
  264. u16 = np.frombuffer(raw, dtype=np.uint16).copy()
  265. return (u16.astype(np.uint32) << 16).view(np.float32).copy()
  266. u8 = np.frombuffer(raw, dtype=np.uint8).copy()
  267. return gguf_quants.dequantize(u8, qt).ravel().astype(np.float32)
  268.  
  269. def f32_to_bytes_aligned(f32, qt, raw_chunk_size):
  270. f32 = np.nan_to_num(f32, nan=0.0, posinf=0.0, neginf=0.0)
  271. if qt == GGMLQuantizationType.F32:
  272. r = f32.astype(np.float32).tobytes()
  273. elif qt == GGMLQuantizationType.F16:
  274. r = f32.astype(np.float16).tobytes()
  275. elif qt == GGMLQuantizationType.BF16:
  276. u32 = f32.astype(np.float32).view(np.uint32)
  277. r = (u32 >> 16).astype(np.uint16).tobytes()
  278. elif qt in _QUANTIZE_SUPPORTED:
  279. be = quant_block_elements(qt)
  280. n = align_down(len(f32), be)
  281. if n == 0: return None
  282. try:
  283. r = gguf_quants.quantize(f32[:n], qt).ravel().tobytes()
  284. except Exception:
  285. return None
  286. else:
  287. return None
  288. return r if len(r) == raw_chunk_size else None
  289.  
  290. # ============================================================================
  291. # DOWNLOAD
  292. # ============================================================================
  293.  
  294. def download(spec, label):
  295. print(f"[DL] {label}: {spec['repo']} / {spec['file']}")
  296. path = hf_hub_download(repo_id=spec["repo"], filename=spec["file"],
  297. local_dir=WORK_DIR)
  298. print(f"[DL] {label}: {file_size_gb(path):.2f} GB -> {path}")
  299. return path
  300.  
  301. # ============================================================================
  302. # LLAMA-QUANTIZE
  303. # ============================================================================
  304.  
  305. LLAMA_BIN_DIR = os.path.join(WORK_DIR, "llama-bin")
  306.  
  307. def get_llama_quantize():
  308. for root, dirs, files in os.walk(LLAMA_BIN_DIR):
  309. for f in files:
  310. if f == "llama-quantize":
  311. p = os.path.join(root, f)
  312. if os.access(p, os.X_OK):
  313. print(f"[BIN] Already available: {p}")
  314. return p
  315.  
  316. print(f"[BIN] Downloading Vulkan build...")
  317. t0 = time.time()
  318. tar_path = os.path.join(WORK_DIR, LLAMA_TAR)
  319.  
  320. downloaded = False
  321. for cmd in [
  322. ["wget", "-q", "--show-progress", "-O", tar_path, LLAMA_URL],
  323. ["curl", "-sL", "-o", tar_path, LLAMA_URL],
  324. ]:
  325. try:
  326. subprocess.check_call(cmd, timeout=300)
  327. if os.path.isfile(tar_path) and os.path.getsize(tar_path) > 10240:
  328. downloaded = True; break
  329. except Exception:
  330. continue
  331.  
  332. if not downloaded:
  333. print("[BIN] ERROR: Download failed."); return None
  334.  
  335. os.makedirs(LLAMA_BIN_DIR, exist_ok=True)
  336. for strip in ["--strip-components=1", None]:
  337. try:
  338. cmd = ["tar", "xzf", tar_path, "-C", LLAMA_BIN_DIR]
  339. if strip: cmd.append(strip)
  340. subprocess.check_call(cmd); break
  341. except subprocess.CalledProcessError:
  342. continue
  343.  
  344. if os.path.isfile(tar_path): os.remove(tar_path)
  345.  
  346. for root, dirs, files in os.walk(LLAMA_BIN_DIR):
  347. for f in files:
  348. if f == "llama-quantize":
  349. binary = os.path.join(root, f)
  350. os.chmod(binary, 0o755)
  351. print(f"[BIN] Ready in {time.time()-t0:.1f}s")
  352. return binary
  353.  
  354. print("[BIN] ERROR: llama-quantize not found."); return None
  355.  
  356. def run_llama_quantize(input_path, output_path, quant_type):
  357. binary = get_llama_quantize()
  358. if binary is None: return None
  359.  
  360. bin_dir = os.path.dirname(binary)
  361. env = os.environ.copy()
  362. ld = env.get("LD_LIBRARY_PATH", "")
  363. env["LD_LIBRARY_PATH"] = f"{bin_dir}:{ld}" if ld else bin_dir
  364.  
  365. cmd = [binary, "--allow-requantize", input_path, output_path, quant_type]
  366. print(f"[QUANT] {' '.join(cmd)}")
  367. t0 = time.time()
  368.  
  369. try:
  370. proc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
  371. stderr=subprocess.STDOUT, text=True, env=env, bufsize=1)
  372. for line in proc.stdout:
  373. line = line.rstrip()
  374. if line: print(f" {line}")
  375. proc.wait()
  376. if proc.returncode != 0:
  377. print(f"[QUANT] ERROR: exit {proc.returncode}"); return None
  378. except Exception as e:
  379. print(f"[QUANT] ERROR: {e}"); return None
  380.  
  381. if not os.path.isfile(output_path):
  382. print("[QUANT] ERROR: Output not created."); return None
  383.  
  384. elapsed = time.time() - t0
  385. print(f"[QUANT] Done in {elapsed:.0f}s ({elapsed/60:.1f} min)")
  386. return output_path
  387.  
  388. # ============================================================================
  389. # STAGE 1: CROSS-QUANT MERGE (Q4_K_M sources → float32 → Q4_0 output)
  390. # ============================================================================
  391.  
  392. def is_finetune_weight(name):
  393. return any(name.endswith(s) for s in FINETUNE_SUFFIXES)
  394.  
  395. def is_protected(name):
  396. return any(p in name for p in PROTECTED_SUBSTRINGS)
  397.  
  398. def classify_tensor_merge(name, ti_out, ti_b, ti_c):
  399. if is_protected(name): return "protect", "structural/norm/embd"
  400. if not is_finetune_weight(name): return "protect", "not fine-tunable"
  401. if ti_b is None or ti_c is None: return "skip", "missing in B/C"
  402. if ti_out["n_elements"] != ti_b["n_elements"]:
  403. return "skip", f"elem mismatch {ti_out['n_elements']}��{ti_b['n_elements']}"
  404. if ti_b["n_elements"] != ti_c["n_elements"]:
  405. return "skip", f"elem mismatch B≠C"
  406. return "merge", "fine-tunable"
  407.  
  408. def merge_tensor_cross(f_out, ti_out, f_b, ti_b, f_c, ti_c, weight,
  409. qt_out, qt_b, qt_c):
  410. total_elems = ti_out["n_elements"]
  411.  
  412. be_out = quant_block_elements(qt_out)
  413. bb_out = quant_block_bytes(qt_out)
  414. be_b = quant_block_elements(qt_b)
  415. bb_b = quant_block_bytes(qt_b)
  416. be_c = quant_block_elements(qt_c)
  417. bb_c = quant_block_bytes(qt_c)
  418.  
  419. lcm_bc = be_b * be_c // gcd(be_b, be_c)
  420. lcm_all = lcm_bc * be_out // gcd(lcm_bc, be_out)
  421.  
  422. chunk_elems = align_down(MERGE_CHUNK_ELEMS, lcm_all)
  423. if chunk_elems <= 0: chunk_elems = lcm_all
  424.  
  425. changed = 0; failed = 0; elem_off = 0
  426.  
  427. while elem_off < total_elems:
  428. n = min(chunk_elems, total_elems - elem_off)
  429. n = align_down(n, lcm_all)
  430. if n <= 0: break
  431.  
  432. out_boff = elems_to_raw(elem_off, qt_out)
  433. out_bsz = elems_to_raw(n, qt_out)
  434. b_boff = elems_to_raw(elem_off, qt_b)
  435. b_bsz = elems_to_raw(n, qt_b)
  436. c_boff = elems_to_raw(elem_off, qt_c)
  437. c_bsz = elems_to_raw(n, qt_c)
  438.  
  439. ra = read_chunk(f_out, ti_out, out_boff, out_bsz)
  440. rb = read_chunk(f_b, ti_b, b_boff, b_bsz)
  441. rc = read_chunk(f_c, ti_c, c_boff, c_bsz)
  442.  
  443. if len(ra) != out_bsz or len(rb) != b_bsz or len(rc) != c_bsz:
  444. failed += 1; elem_off += n; continue
  445.  
  446. try:
  447. fa = bytes_to_f32(ra, qt_out)
  448. fb = bytes_to_f32(rb, qt_b)
  449. fc = bytes_to_f32(rc, qt_c)
  450. except Exception:
  451. failed += 1; del ra, rb, rc; elem_off += n; continue
  452.  
  453. k = min(len(fa), len(fb), len(fc))
  454. k = align_down(k, be_out)
  455. if k == 0:
  456. failed += 1; del ra, rb, rc, fa, fb, fc; elem_off += n; continue
  457.  
  458. # *** MERGE IN FLOAT32 — NO PRECISION LOSS ***
  459. merged = fa[:k] + (fb[:k] - fc[:k]) * np.float32(weight)
  460. del fa, fb, fc
  461.  
  462. new_raw = f32_to_bytes_aligned(merged, qt_out, out_bsz)
  463. del merged
  464.  
  465. if new_raw is not None:
  466. old_arr = np.frombuffer(ra, dtype=np.uint8)
  467. new_arr = np.frombuffer(new_raw, dtype=np.uint8)
  468. changed += int(np.sum(old_arr != new_arr))
  469. write_chunk(f_out, ti_out, out_boff, new_raw)
  470. del old_arr, new_arr
  471. else:
  472. failed += 1
  473.  
  474. del ra, rb, rc
  475. if new_raw is not None: del new_raw
  476. elem_off += n
  477.  
  478. return changed, failed
  479.  
  480. def run_merge(q4_0_path, path_b, path_c, weight):
  481. t0 = time.time()
  482.  
  483. print("[MERGE] Parsing structures...")
  484. info_out, n_out = parse_gguf_structure(q4_0_path)
  485. info_b, n_b = parse_gguf_structure(path_b)
  486. info_c, n_c = parse_gguf_structure(path_c)
  487.  
  488. print(f"[MERGE] Output (Q4_0): {n_out} tensors ({file_size_gb(q4_0_path):.2f} GB)")
  489. print(f"[MERGE] B (Q4_K_M): {n_b} tensors ({file_size_gb(path_b):.2f} GB)")
  490. print(f"[MERGE] C (Q4_K_M): {n_c} tensors ({file_size_gb(path_c):.2f} GB)")
  491.  
  492. plan = {}
  493. for name in sorted(info_out.keys()):
  494. ti_out = info_out[name]
  495. ti_b = info_b.get(name)
  496. ti_c = info_c.get(name)
  497. action, reason = classify_tensor_merge(name, ti_out, ti_b, ti_c)
  498. plan[name] = (action, reason)
  499.  
  500. n_merge = sum(1 for v in plan.values() if v[0] == "merge")
  501. n_protect = sum(1 for v in plan.values() if v[0] == "protect")
  502. n_skip = sum(1 for v in plan.values() if v[0] == "skip")
  503.  
  504. print(f"\n[MERGE] MERGE={n_merge} PROTECT={n_protect} SKIP={n_skip}")
  505.  
  506. if n_skip > 0:
  507. for name, (action, reason) in sorted(plan.items()):
  508. if action == "skip":
  509. print(f" SKIP: {name:<48} {reason}")
  510.  
  511. if n_merge == 0:
  512. print("[MERGE] Nothing to merge."); return {}
  513.  
  514. f_out = open(q4_0_path, "r+b")
  515. f_b = open(path_b, "rb")
  516. f_c = open(path_c, "rb")
  517.  
  518. print(f"\n[MERGE] output = A + (B - C) * {weight}")
  519. print(f"[MERGE] All arithmetic in float32")
  520. print(f"{'#':>4} {'Tensor':<50} {'Out':<6} {'Src':<8} "
  521. f"{'Size':>9} {'Action':<10} Detail")
  522. print("-" * 115)
  523.  
  524. stats = {"merged":0,"protected":0,"skipped":0,
  525. "total_changed":0,"identical":0,"failed":0}
  526.  
  527. for idx, name in enumerate(sorted(info_out.keys())):
  528. ti_out = info_out[name]
  529. qt_out_name = ti_out["qtype_name"]
  530. sz = ti_out["size"]
  531. action, reason = plan[name]
  532.  
  533. if action != "merge":
  534. stats["protected" if action == "protect" else "skipped"] += 1
  535. print(f"{idx+1:>4} {name:<50} {qt_out_name:<6} {'--':<8} "
  536. f"{fmt_bytes(sz):>9} {action.upper():<10} {reason}")
  537. continue
  538.  
  539. ti_b = info_b[name]; ti_c = info_c[name]
  540. qt_out = ti_out["qtype"]; qt_b = ti_b["qtype"]; qt_c = ti_c["qtype"]
  541.  
  542. changed, failed = merge_tensor_cross(
  543. f_out, ti_out, f_b, ti_b, f_c, ti_c, weight, qt_out, qt_b, qt_c)
  544.  
  545. stats["merged"] += 1; stats["total_changed"] += changed; stats["failed"] += failed
  546.  
  547. src_qt = ti_b["qtype_name"]
  548. if changed > 0:
  549. f_s = f" ({failed} fail)" if failed else ""
  550. print(f"{idx+1:>4} {name:<50} {qt_out_name:<6} {src_qt:<8} "
  551. f"{fmt_bytes(sz):>9} {'MERGED':<10} {changed:,}B{f_s}")
  552. else:
  553. stats["identical"] += 1
  554. print(f"{idx+1:>4} {name:<50} {qt_out_name:<6} {src_qt:<8} "
  555. f"{fmt_bytes(sz):>9} {'MERGED':<10} (identical)")
  556.  
  557. if (idx + 1) % 50 == 0: gc.collect()
  558.  
  559. flush_and_close(f_out, "output"); f_b.close(); f_c.close()
  560. elapsed = time.time() - t0
  561.  
  562. print(f"\n{'='*75}")
  563. print(f"MERGE SUMMARY: {stats['merged']} merged, {stats['protected']} protected, "
  564. f"{stats['identical']} identical, {elapsed:.1f}s")
  565. if stats['failed']: print(f" Failed chunks: {stats['failed']}")
  566. print(f"Output: {file_size_gb(q4_0_path):.2f} GB")
  567. print(f"{'='*75}")
  568.  
  569. return stats
  570.  
  571. # ============================================================================
  572. # VERIFY
  573. # ============================================================================
  574.  
  575. def verify_output(output_path):
  576. print(f"\n[VERIFY] {os.path.basename(output_path)}...")
  577. try:
  578. info_out, n_out = parse_gguf_structure(output_path)
  579. print(f"[VERIFY] {n_out} tensors parsed.")
  580. except Exception as e:
  581. print(f"[VERIFY] ERROR: {e}"); return False
  582.  
  583. f = open(output_path, "rb")
  584. ok = warn = 0
  585. for name in sorted(info_out.keys())[:30]:
  586. ti = info_out[name]; cs = min(4096, ti["size"])
  587. raw = read_chunk(f, ti, 0, cs)
  588. arr = np.frombuffer(raw, dtype=np.uint8)
  589. if arr.sum() == 0 and cs > 64:
  590. warn += 1; print(f" WARNING: '{name}' all zeros!")
  591. else: ok += 1
  592. del raw, arr
  593. f.close()
  594. print(f"[VERIFY] {ok} OK" + (f", {warn} WARNINGS" if warn else ""))
  595. return warn == 0
  596.  
  597. # ============================================================================
  598. # MAIN PIPELINE
  599. # ============================================================================
  600.  
  601. def main():
  602. print("=" * 75)
  603. print("GGUF PIPELINE: Q4_K_M → float32 merge → Q4_0 → Q4_K_M")
  604. print()
  605. print(f" Model A: {MODEL_A['repo']} / {MODEL_A['file']}")
  606. print(f" Model B: {MODEL_B['repo']} / {MODEL_B['file']}")
  607. print(f" Model C: {MODEL_C['repo']} / {MODEL_C['file']}")
  608. print(f" Merge weight: {MERGE_WEIGHT}")
  609. print(f" Final quant: {FINAL_QTYPE if FINAL_QTYPE else 'None (keep Q4_0)'}")
  610. print(f" Disk free: {disk_free_gb():.1f} GB")
  611. print(f" RAM used: {ram_used_gb():.2f} GB")
  612. print()
  613. print(" Data flow (minimal loss):")
  614. print(" Q4_K_M(A) ──llama-quantize──→ Q4_0(A') [1 requant]")
  615. print(" Q4_K_M(B,C) ──dequant──→ float32 ──merge──→ float32 [lossless]")
  616. print(" float32 ──quantize──→ Q4_0(merged) [1 quantization]")
  617. print("=" * 75)
  618.  
  619. check_gpu()
  620.  
  621. # ================================================================
  622. # STAGE 0: Download A, convert to Q4_0, free disk
  623. # ================================================================
  624. print("\n" + "=" * 75)
  625. print("STAGE 0: Download A (Q4_K_M) → convert Q4_0")
  626. print("=" * 75)
  627.  
  628. path_a = download(MODEL_A, "A (base)")
  629. a_gb = file_size_gb(path_a)
  630. print(f"[FS] A: {a_gb:.2f} GB | Disk free: {disk_free_gb():.1f} GB")
  631.  
  632. print(f"\n[CONV] Converting A: Q4_K_M → Q4_0 via llama-quantize...")
  633. result = run_llama_quantize(path_a, MERGED_PATH, "Q4_0")
  634. if result is None:
  635. sys.exit("[ERROR] Q4_0 conversion failed.")
  636.  
  637. merged_gb = file_size_gb(MERGED_PATH)
  638. print(f"[CONV] {a_gb:.2f} GB → {merged_gb:.2f} GB")
  639.  
  640. safe_remove(path_a, protect=MERGED_PATH)
  641. print(f"[FS] Disk free: {disk_free_gb():.1f} GB")
  642.  
  643. # ================================================================
  644. # STAGE 1: Download B, C → cross-quant merge → delete B, C
  645. # ================================================================
  646. print("\n" + "=" * 75)
  647. print("STAGE 1: Download B, C → merge into Q4_0")
  648. print(" Dequant Q4_K_M → float32 → merge → requant Q4_0")
  649. print("=" * 75)
  650.  
  651. path_b = download(MODEL_B, "B (fine-tuned)")
  652. path_c = download(MODEL_C, "C (reference)")
  653. print(f"[FS] Disk free: {disk_free_gb():.1f} GB")
  654.  
  655. run_merge(MERGED_PATH, path_b, path_c, MERGE_WEIGHT)
  656. gc.collect()
  657.  
  658. verify_output(MERGED_PATH)
  659.  
  660. print("\n[CLEANUP] Removing B, C...")
  661. safe_remove(path_c, protect=MERGED_PATH)
  662. safe_remove(path_b, protect=MERGED_PATH)
  663. gc.collect()
  664. print(f"[FS] Disk free: {disk_free_gb():.1f} GB")
  665.  
  666. # ================================================================
  667. # STAGE 2: Final quantization Q4_0 → Q4_K_M
  668. # ================================================================
  669. final_path = MERGED_PATH
  670.  
  671. if FINAL_QTYPE:
  672. print("\n" + "=" * 75)
  673. print(f"STAGE 2: QUANTIZE Q4_0 → {FINAL_QTYPE}")
  674. print("=" * 75)
  675.  
  676. base = os.path.splitext(os.path.basename(MERGED_PATH))[0]
  677. for sfx in ["-Q4_0","-Q4_K_M","-merged"]:
  678. if base.endswith(sfx): base = base[:-len(sfx)]
  679.  
  680. output_name = f"{base}-{FINAL_QTYPE}.gguf"
  681. output_path = os.path.join(WORK_DIR, output_name)
  682.  
  683. input_gb = file_size_gb(MERGED_PATH)
  684. print(f"[QUANT] Input: {input_gb:.2f} GB | Disk free: {disk_free_gb():.1f} GB")
  685.  
  686. result = run_llama_quantize(MERGED_PATH, output_path, FINAL_QTYPE)
  687.  
  688. if result and os.path.isfile(output_path):
  689. final_path = output_path
  690. output_gb = file_size_gb(output_path)
  691. print(f"[QUANT] {input_gb:.2f} GB → {output_gb:.2f} GB")
  692.  
  693. safe_remove(MERGED_PATH, protect=final_path)
  694. verify_output(final_path)
  695. else:
  696. print("[QUANT] Failed. Keeping Q4_0.")
  697. else:
  698. print("\n[INFO] Final quantize disabled. Keeping Q4_0.")
  699.  
  700. # ================================================================
  701. # DONE
  702. # ================================================================
  703. print("\n" + "=" * 75)
  704. print("PIPELINE COMPLETE")
  705. print("=" * 75)
  706. print(f"\n[FS] {WORK_DIR}:")
  707. for fn in sorted(os.listdir(WORK_DIR)):
  708. fp = os.path.join(WORK_DIR, fn)
  709. if os.path.isfile(fp):
  710. print(f" {fn} ({file_size_gb(fp):.2f} GB)")
  711. print(f"\n Final: {final_path}")
  712. print(f" Size: {file_size_gb(final_path):.2f} GB")
  713. print(f" Disk free: {disk_free_gb():.1f} GB")
  714. print(f" RAM used: {ram_used_gb():.2f} GB")
  715.  
  716. main()
Advertisement
Add Comment
Please, Sign In to add comment