LuffyTheFox

Untitled

Apr 5th, 2026
183
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 31.95 KB | None | 0 0
  1. # ============================================================================
  2. # GGUF Add-Difference Merge — Disk-Optimized
  3. #
  4. # output = A + (B - C) * weight
  5. #
  6. # Pipeline:
  7. # 1) Download B and C
  8. # 2) Compute delta = B - C → GGUF (only actually changed tensors)
  9. # 3) Delete B and C
  10. # 4) Download A
  11. # 5) Apply output = A + delta * weight (in-place rename, no copy)
  12. # 6) Delete delta (keep only merged output)
  13. #
  14. # • Delta is standard GGUF in original quantization format
  15. # • Only tensors where raw bytes B ≠ C are included
  16. # • Two-pass delta: pass1=raw byte compare, pass2=compute+write diffs
  17. # • Streaming GGUF write: one tensor in RAM at a time
  18. # • Punch-hole: processed tensors in B/C are deallocated via fallocate
  19. # • In-place apply: A is renamed to output, no disk copy needed
  20. # ============================================================================
  21.  
  22. import subprocess, sys, os, gc, time, shutil, ctypes, ctypes.util
  23. import numpy as np
  24.  
  25. WORK_DIR = "/content/gguf_workspace"
  26. os.makedirs(WORK_DIR, exist_ok=True)
  27.  
  28. def install():
  29. for pkg in ["huggingface_hub[hf_xet]", "gguf", "hf_xet"]:
  30. subprocess.check_call(
  31. [sys.executable, "-m", "pip", "install", "-q", pkg],
  32. stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
  33. )
  34. print("[OK] Dependencies installed.")
  35.  
  36. install()
  37.  
  38. from huggingface_hub import hf_hub_download
  39. from gguf import GGUFReader, GGUFWriter
  40. from gguf.constants import GGMLQuantizationType, GGML_QUANT_SIZES
  41. from gguf import quants as gguf_quants
  42.  
  43. # ── Enable hf_xet for downloads ──────────────────────────────────────────
  44. try:
  45. import hf_xet
  46. print("[OK] hf_xet backend loaded — Xet-accelerated downloads enabled.")
  47. except ImportError:
  48. print("[WARN] hf_xet not available — falling back to default download.")
  49.  
  50. # ── Punch-hole support (Linux fallocate) ─────────────────────────────────
  51. _FALLOC_FL_PUNCH_HOLE = 0x02
  52. _FALLOC_FL_KEEP_SIZE = 0x01
  53. _libc = None
  54. _punch_hole_supported = False
  55.  
  56. def _init_punch_hole():
  57. global _libc, _punch_hole_supported
  58. if sys.platform != "linux":
  59. print("[INFO] Punch-hole: not Linux — disabled.")
  60. return
  61. try:
  62. _libc_name = ctypes.util.find_library("c")
  63. if _libc_name:
  64. _libc = ctypes.CDLL(_libc_name, use_errno=True)
  65. tf = os.path.join(WORK_DIR, "_punch_test")
  66. with open(tf, "wb") as f:
  67. f.write(b"\xff" * 8192)
  68. fd = os.open(tf, os.O_RDWR)
  69. ret = _libc.fallocate(
  70. ctypes.c_int(fd),
  71. ctypes.c_int(_FALLOC_FL_PUNCH_HOLE | _FALLOC_FL_KEEP_SIZE),
  72. ctypes.c_longlong(0),
  73. ctypes.c_longlong(4096),
  74. )
  75. os.close(fd)
  76. os.remove(tf)
  77. if ret == 0:
  78. _punch_hole_supported = True
  79. print("[OK] Punch-hole (fallocate PUNCH_HOLE): supported.")
  80. else:
  81. print(f"[INFO] Punch-hole: fallocate returned {ret} — disabled.")
  82. else:
  83. print("[INFO] Punch-hole: libc not found — disabled.")
  84. except Exception as e:
  85. print(f"[INFO] Punch-hole: init failed ({e}) — disabled.")
  86.  
  87. _init_punch_hole()
  88.  
  89. def punch_hole(filepath, offset, length):
  90. if not _punch_hole_supported or _libc is None or length <= 0:
  91. return 0
  92. try:
  93. fd = os.open(filepath, os.O_RDWR)
  94. try:
  95. BS = 4096
  96. aligned_start = ((offset + BS - 1) // BS) * BS
  97. aligned_end = ((offset + length) // BS) * BS
  98. if aligned_end <= aligned_start:
  99. return 0
  100. punch_len = aligned_end - aligned_start
  101. ret = _libc.fallocate(
  102. ctypes.c_int(fd),
  103. ctypes.c_int(_FALLOC_FL_PUNCH_HOLE | _FALLOC_FL_KEEP_SIZE),
  104. ctypes.c_longlong(aligned_start),
  105. ctypes.c_longlong(punch_len),
  106. )
  107. if ret == 0:
  108. return punch_len
  109. return 0
  110. finally:
  111. os.close(fd)
  112. except Exception:
  113. return 0
  114.  
  115. def file_blocks_kb(filepath):
  116. try:
  117. st = os.stat(filepath)
  118. return (st.st_blocks * 512) / 1024
  119. except Exception:
  120. return -1
  121.  
  122. # ── Patch IQ block sizes ─────────────────────────────────────────────────
  123. _IQ = {
  124. "IQ1_S":(256,50),"IQ1_M":(256,56),"IQ2_XXS":(256,66),"IQ2_XS":(256,74),
  125. "IQ2_S":(256,82),"IQ3_XXS":(256,98),"IQ3_S":(256,110),
  126. "IQ4_NL":(32,18),"IQ4_XS":(256,136),
  127. }
  128. for _n,_g in _IQ.items():
  129. _qt = getattr(GGMLQuantizationType, _n, None)
  130. if _qt is not None and _qt not in GGML_QUANT_SIZES:
  131. GGML_QUANT_SIZES[_qt] = _g
  132.  
  133. # ── Probe quantize support ──────────────────────────────────────────────
  134. _QUANTIZE_SUPPORTED = set()
  135.  
  136. def _probe_quantize():
  137. for qt in GGMLQuantizationType:
  138. if qt in (GGMLQuantizationType.F32, GGMLQuantizationType.F16,
  139. GGMLQuantizationType.BF16):
  140. _QUANTIZE_SUPPORTED.add(qt)
  141. elif qt in GGML_QUANT_SIZES:
  142. be, _ = GGML_QUANT_SIZES[qt]
  143. try:
  144. r = gguf_quants.quantize(np.zeros(be, np.float32), qt)
  145. if r is not None and len(r.ravel().tobytes()) > 0:
  146. _QUANTIZE_SUPPORTED.add(qt)
  147. except Exception: pass
  148.  
  149. _probe_quantize()
  150. print(f"[INFO] quantize() supported: {sorted([q.name for q in _QUANTIZE_SUPPORTED])}")
  151.  
  152. # ── Check streaming GGUF write ──────────────────────────────────────────
  153. _STREAMING = (
  154. hasattr(GGUFWriter, 'add_tensor_info') and
  155. hasattr(GGUFWriter, 'write_tensor_data') and
  156. hasattr(GGUFWriter, 'write_ti_data_to_file')
  157. )
  158. print(f"[INFO] GGUF streaming write: {'YES' if _STREAMING else 'NO (buffered)'}")
  159.  
  160. # ============================================================================
  161. # CONFIGURATION
  162. # ============================================================================
  163. MODEL_A = {
  164. "repo": "samuelcardillo/Qwen3.5-35B-A3B-Claude-4.6-Opus-Reasoning-Distilled-GGUF",
  165. "file": "Qwen3.5-35B-A3B-Claude-4.6-Opus-Reasoning-Distilled-Q8_0.gguf",
  166. }
  167. MODEL_B = {
  168. "repo": "HauhauCS/Qwen3.5-35B-A3B-Uncensored-HauhauCS-Aggressive",
  169. "file": "Qwen3.5-35B-A3B-Uncensored-HauhauCS-Aggressive-Q8_0.gguf",
  170. }
  171. MODEL_C = {
  172. "repo": "unsloth/Qwen3.5-35B-A3B-GGUF",
  173. "file": "Qwen3.5-35B-A3B-Q8_0.gguf",
  174. }
  175.  
  176. MERGE_WEIGHT = 1.0
  177. OUTPUT_NAME = "Qwen3.5-35B-A3B-Q8_0-merged.gguf"
  178. OUTPUT_PATH = os.path.join(WORK_DIR, OUTPUT_NAME)
  179. DELTA_PATH = os.path.join(WORK_DIR, "_delta.gguf")
  180. CHUNK_BYTES = 4 * 1024 * 1024
  181.  
  182. # ============================================================================
  183. # TENSOR CLASSIFICATION
  184. # ============================================================================
  185. FINETUNE_SUFFIXES = (
  186. "attn_q.weight","attn_k.weight","attn_v.weight","attn_output.weight",
  187. "ssm_in.weight","ssm_out.weight","ssm_a.weight",
  188. "ssm_d.weight","ssm_dt.weight",
  189. "ffn_gate.weight","ffn_up.weight","ffn_down.weight",
  190. "ffn_gate_exps.weight","ffn_up_exps.weight","ffn_down_exps.weight",
  191. "ffn_gate_inp.weight","ffn_gate_shexp.weight",
  192. "ffn_up_shexp.weight","ffn_down_shexp.weight",
  193. "attn_gate.weight",
  194. )
  195.  
  196. PROTECTED_SUBSTRINGS = (
  197. "token_embd","output.weight","_norm",".bias",
  198. "rope_freqs","pos_embd",
  199. "v.blk.","v.patch_embd","v.position_embd",
  200. "v.post_ln","v.pre_ln","v.output_norm","v.proj",
  201. "mm.","mtp.",
  202. )
  203.  
  204. FLOAT_QTYPES = {
  205. GGMLQuantizationType.F32, GGMLQuantizationType.F16, GGMLQuantizationType.BF16,
  206. }
  207.  
  208. # ============================================================================
  209. # UTILITIES
  210. # ============================================================================
  211.  
  212. def disk_free_gb(p=WORK_DIR):
  213. st = os.statvfs(p); return (st.f_bavail * st.f_frsize) / 1073741824
  214.  
  215. def file_size_gb(p):
  216. return os.path.getsize(p) / 1073741824 if os.path.exists(p) else 0.0
  217.  
  218. def fmt_bytes(n):
  219. for u in ("B","KB","MB","GB"):
  220. if n < 1024: return f"{n:.1f} {u}"
  221. n /= 1024
  222. return f"{n:.1f} TB"
  223.  
  224. def safe_remove(path, protect=None):
  225. try:
  226. a = os.path.abspath(path)
  227. if protect:
  228. if isinstance(protect, str): protect = [protect]
  229. for p in protect:
  230. if a == os.path.abspath(p): return
  231. if os.path.isfile(a):
  232. sz = file_size_gb(a)
  233. os.remove(a)
  234. print(f"[CLEANUP] Removed {os.path.basename(a)} ({sz:.2f} GB)")
  235. except OSError: pass
  236.  
  237. def flush_and_close(f, label="file"):
  238. f.flush(); os.fsync(f.fileno()); f.close()
  239. print(f"[FLUSH] {label}: flush + fsync done.")
  240.  
  241. def block_align(qt):
  242. if qt in FLOAT_QTYPES:
  243. return {GGMLQuantizationType.F32:4,GGMLQuantizationType.F16:2,
  244. GGMLQuantizationType.BF16:2}[qt]
  245. if qt in GGML_QUANT_SIZES: return GGML_QUANT_SIZES[qt][1]
  246. return 1
  247.  
  248. def align_down(v, a):
  249. return (v // a) * a if a > 0 else v
  250.  
  251. def print_workspace():
  252. print(f"\n[FS] {WORK_DIR}:")
  253. for fn in sorted(os.listdir(WORK_DIR)):
  254. fp = os.path.join(WORK_DIR, fn)
  255. if os.path.isfile(fp):
  256. logical = file_size_gb(fp)
  257. actual_kb = file_blocks_kb(fp)
  258. if actual_kb >= 0:
  259. actual_gb = actual_kb / (1024*1024)
  260. saved = logical - actual_gb
  261. extra = f" (on-disk: {actual_gb:.2f} GB, saved: {saved:.2f} GB)" if saved > 0.01 else ""
  262. else:
  263. extra = ""
  264. print(f" {fn} ({logical:.2f} GB){extra}")
  265. print(f" Disk free: {disk_free_gb():.1f} GB")
  266.  
  267. # ============================================================================
  268. # CLASSIFICATION
  269. # ============================================================================
  270.  
  271. def is_finetune_weight(name):
  272. return any(name.endswith(s) for s in FINETUNE_SUFFIXES)
  273.  
  274. def is_protected(name):
  275. return any(p in name for p in PROTECTED_SUBSTRINGS)
  276.  
  277. def classify_bc(name, ti_b, ti_c):
  278. if is_protected(name): return "protect", "structural/norm/embd"
  279. if not is_finetune_weight(name): return "protect", "not fine-tunable"
  280. if ti_c is None: return "skip", "missing in C"
  281. if ti_b["size"] != ti_c["size"]:
  282. return "skip", f"size mismatch B={ti_b['size']:,} C={ti_c['size']:,}"
  283. if ti_b["qtype_name"] != ti_c["qtype_name"]:
  284. return "skip", f"type mismatch B={ti_b['qtype_name']} C={ti_c['qtype_name']}"
  285. if ti_b["n_elements"] != ti_c["n_elements"]:
  286. return "skip", f"elem mismatch"
  287. if ti_b["qtype"] not in _QUANTIZE_SUPPORTED:
  288. return "skip", f"{ti_b['qtype_name']} no quantize()"
  289. return "merge", "fine-tunable"
  290.  
  291. # ============================================================================
  292. # GGUF PARSING
  293. # ============================================================================
  294.  
  295. def parse_gguf(path):
  296. reader = GGUFReader(path)
  297. tensors = {}
  298. for t in reader.tensors:
  299. tensors[t.name] = {
  300. "offset": int(t.data_offset), "size": int(t.n_bytes),
  301. "qtype": t.tensor_type, "qtype_name": t.tensor_type.name,
  302. "n_elements": int(t.n_elements), "shape": list(t.shape),
  303. }
  304. n = len(tensors); del reader; gc.collect()
  305. return tensors, n
  306.  
  307. # ============================================================================
  308. # RAW I/O
  309. # ============================================================================
  310.  
  311. def read_chunk(f, ti, off, sz):
  312. actual = min(sz, ti["size"] - off)
  313. if actual <= 0: return b""
  314. f.seek(ti["offset"] + off); return f.read(actual)
  315.  
  316. def write_chunk(f, ti, off, data):
  317. f.seek(ti["offset"] + off); f.write(data)
  318.  
  319. def raw_equal(f1, t1, f2, t2):
  320. if t1["size"] != t2["size"]: return False
  321. sz = t1["size"]; off = 0
  322. while off < sz:
  323. n = min(CHUNK_BYTES, sz - off)
  324. f1.seek(t1["offset"]+off); d1 = f1.read(n)
  325. f2.seek(t2["offset"]+off); d2 = f2.read(n)
  326. if d1 != d2: return False
  327. off += n
  328. return True
  329.  
  330. # ============================================================================
  331. # DEQUANT / QUANT
  332. # ============================================================================
  333.  
  334. def to_f32(raw, qt):
  335. if qt == GGMLQuantizationType.F32:
  336. return np.frombuffer(raw, np.float32).copy()
  337. if qt == GGMLQuantizationType.F16:
  338. return np.frombuffer(raw, np.float16).astype(np.float32)
  339. if qt == GGMLQuantizationType.BF16:
  340. u = np.frombuffer(raw, np.uint16).copy()
  341. return (u.astype(np.uint32) << 16).view(np.float32).copy()
  342. return gguf_quants.dequantize(np.frombuffer(raw, np.uint8).copy(), qt
  343. ).ravel().astype(np.float32)
  344.  
  345. def from_f32(f, qt, expected):
  346. f = np.nan_to_num(f, nan=0., posinf=0., neginf=0.)
  347. if qt == GGMLQuantizationType.F32:
  348. r = f.astype(np.float32).tobytes()
  349. elif qt == GGMLQuantizationType.F16:
  350. r = f.astype(np.float16).tobytes()
  351. elif qt == GGMLQuantizationType.BF16:
  352. r = (f.astype(np.float32).view(np.uint32) >> 16).astype(np.uint16).tobytes()
  353. elif qt in _QUANTIZE_SUPPORTED:
  354. be, _ = GGML_QUANT_SIZES[qt]
  355. n = align_down(len(f), be)
  356. if n == 0: return None
  357. try: r = gguf_quants.quantize(f[:n], qt).ravel().tobytes()
  358. except Exception: return None
  359. else: return None
  360. return r if len(r) == expected else None
  361.  
  362. def quant_nd(f, qt):
  363. f = np.nan_to_num(f, nan=0., posinf=0., neginf=0.).astype(np.float32)
  364. if qt == GGMLQuantizationType.F32: return f.view(np.uint8).copy()
  365. if qt == GGMLQuantizationType.F16: return f.astype(np.float16).view(np.uint8).copy()
  366. if qt == GGMLQuantizationType.BF16:
  367. return (f.view(np.uint32) >> 16).astype(np.uint16).view(np.uint8).copy()
  368. if qt in _QUANTIZE_SUPPORTED:
  369. be, _ = GGML_QUANT_SIZES[qt]; n = align_down(len(f), be)
  370. if n == 0: return np.zeros(0, np.uint8)
  371. return gguf_quants.quantize(f[:n], qt).ravel().copy()
  372. return np.zeros(0, np.uint8)
  373.  
  374. # ============================================================================
  375. # DOWNLOAD
  376. # ============================================================================
  377.  
  378. def download(spec, label):
  379. print(f"[DL] {label}: {spec['repo']} / {spec['file']}")
  380. p = hf_hub_download(spec["repo"], spec["file"], local_dir=WORK_DIR)
  381. print(f"[DL] {label}: {file_size_gb(p):.2f} GB -> {p}"); return p
  382.  
  383. # ============================================================================
  384. # COMPUTE ONE TENSOR DIFF → quantized uint8 ndarray
  385. # ============================================================================
  386.  
  387. def tensor_diff(f_b, tb, f_c, tc, qt):
  388. sz = tb["size"]; ba = block_align(qt)
  389. cb = align_down(CHUNK_BYTES, ba)
  390. if cb <= 0: cb = ba
  391. parts = []; off = 0
  392. while off < sz:
  393. act = align_down(min(cb, sz - off), ba)
  394. if act <= 0: break
  395. rb = read_chunk(f_b, tb, off, act)
  396. rc = read_chunk(f_c, tc, off, act)
  397. if len(rb) != act or len(rc) != act: break
  398. try:
  399. d = to_f32(rb, qt) - to_f32(rc, qt)
  400. parts.append(np.nan_to_num(d, nan=0., posinf=0., neginf=0.))
  401. except Exception:
  402. if qt in FLOAT_QTYPES:
  403. eb = {GGMLQuantizationType.F32:4,GGMLQuantizationType.F16:2,
  404. GGMLQuantizationType.BF16:2}[qt]
  405. parts.append(np.zeros(act//eb, np.float32))
  406. elif qt in GGML_QUANT_SIZES:
  407. be, bs = GGML_QUANT_SIZES[qt]
  408. parts.append(np.zeros((act//bs)*be, np.float32))
  409. del rb, rc; off += act
  410. full = np.concatenate(parts) if parts else np.zeros(tb["n_elements"], np.float32)
  411. del parts; r = quant_nd(full, qt); del full; return r
  412.  
  413. # ============================================================================
  414. # STAGE 1: COMPUTE DELTA = B - C → GGUF (only changed tensors)
  415. # + punch holes in B and C after each tensor is processed
  416. # ============================================================================
  417.  
  418. def compute_delta(path_b, path_c, delta_path):
  419. t0 = time.time()
  420. print("\n[DELTA] Parsing B and C...")
  421. ib, nb = parse_gguf(path_b); ic, nc = parse_gguf(path_c)
  422. print(f"[DELTA] B: {nb} tensors C: {nc} tensors")
  423.  
  424. plan = {}; mergeable = []
  425. for nm in sorted(ib.keys()):
  426. a, r = classify_bc(nm, ib[nm], ic.get(nm))
  427. plan[nm] = (a, r)
  428. if a == "merge": mergeable.append(nm)
  429.  
  430. np_ = sum(1 for v in plan.values() if v[0]=="protect")
  431. ns = sum(1 for v in plan.values() if v[0]=="skip")
  432. print(f"[DELTA] Mergeable={len(mergeable)} Protected={np_} Skipped={ns}")
  433. for nm,(a,r) in sorted(plan.items()):
  434. if a=="skip": print(f" SKIP: {nm:<48} {r}")
  435.  
  436. # ── Punch holes for protected/skipped tensors immediately ────────────
  437. if _punch_hole_supported:
  438. total_punched = 0
  439. for nm in sorted(ib.keys()):
  440. act, _ = plan.get(nm, ("protect",""))
  441. if act != "merge":
  442. tb = ib[nm]
  443. p = punch_hole(path_b, tb["offset"], tb["size"])
  444. total_punched += p
  445. tc = ic.get(nm)
  446. if tc is not None:
  447. p = punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
  448. total_punched += p
  449. if total_punched > 0:
  450. print(f"[PUNCH] Freed {fmt_bytes(total_punched)} from "
  451. f"protected/skipped tensors in B and C")
  452. print(f"[PUNCH] Disk free: {disk_free_gb():.1f} GB")
  453.  
  454. if not mergeable:
  455. w = GGUFWriter(delta_path, arch="delta")
  456. w.add_description("Empty delta")
  457. w.write_header_to_file(); w.write_kv_data_to_file()
  458. w.write_ti_data_to_file(); w.close()
  459. print("[DELTA] Empty delta created."); return {}
  460.  
  461. # ── Pass 1: raw byte compare ─────────────────────────────────────────
  462. print(f"\n[DELTA] Pass 1: raw byte compare ({len(mergeable)} tensors)...")
  463. fb = open(path_b,"rb"); fc = open(path_c,"rb")
  464. changed=[]; same=[]
  465. for i,nm in enumerate(mergeable):
  466. eq = raw_equal(fb, ib[nm], fc, ic[nm])
  467. if eq:
  468. same.append(nm)
  469. if _punch_hole_supported:
  470. punch_hole(path_b, ib[nm]["offset"], ib[nm]["size"])
  471. punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
  472. else:
  473. changed.append(nm)
  474. if (i+1)%100==0 or i+1==len(mergeable):
  475. print(f" {i+1}/{len(mergeable)} changed={len(changed)} same={len(same)}"
  476. f" disk_free={disk_free_gb():.1f}GB")
  477.  
  478. fb.close(); fc.close()
  479.  
  480. print(f"[DELTA] Result: {len(changed)} changed, {len(same)} identical")
  481. if _punch_hole_supported and same:
  482. print(f"[PUNCH] Punched {len(same)} identical tensors from B and C")
  483. print(f"[PUNCH] Disk free: {disk_free_gb():.1f} GB")
  484.  
  485. if not changed:
  486. w = GGUFWriter(delta_path, arch="delta")
  487. w.add_description("Empty delta: B==C")
  488. w.write_header_to_file(); w.write_kv_data_to_file()
  489. w.write_ti_data_to_file(); w.close()
  490. print("[DELTA] Empty delta."); return {"changed":0,"unchanged":len(same)}
  491.  
  492. # ── Write delta GGUF ─────────────────────────────────────────────────
  493. w = GGUFWriter(delta_path, arch="delta")
  494. w.add_description(f"Delta (B-C): {len(changed)} changed tensors. "
  495. f"B={MODEL_B['repo']}/{MODEL_B['file']} "
  496. f"C={MODEL_C['repo']}/{MODEL_C['file']}")
  497. w.add_uint32("delta.changed_count", len(changed))
  498. w.add_uint32("delta.unchanged_count", len(same))
  499.  
  500. fb = open(path_b,"rb"); fc = open(path_c,"rb")
  501.  
  502. if _STREAMING:
  503. for nm in changed:
  504. ti = ib[nm]
  505. w.add_tensor_info(name=nm, tensor_shape=ti["shape"],
  506. tensor_dtype=np.float32,
  507. tensor_nbytes=ti["size"], raw_dtype=ti["qtype"])
  508. w.write_header_to_file()
  509. w.write_kv_data_to_file()
  510. w.write_ti_data_to_file()
  511.  
  512. print(f"\n[DELTA] Pass 2: compute + stream {len(changed)} deltas...")
  513. print(f"{'#':>4} {'Tensor':<52} {'Type':<10} {'Size':>9} {'Punched':>10}")
  514. print("-"*95)
  515. total_punched_p2 = 0
  516. for i,nm in enumerate(changed):
  517. d = tensor_diff(fb, ib[nm], fc, ic[nm], ib[nm]["qtype"])
  518. w.write_tensor_data(d)
  519.  
  520. pb = pc = 0
  521. if _punch_hole_supported:
  522. pb = punch_hole(path_b, ib[nm]["offset"], ib[nm]["size"])
  523. pc = punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
  524. total_punched_p2 += pb + pc
  525.  
  526. print(f"{i+1:>4} {nm:<52} {ib[nm]['qtype_name']:<10} "
  527. f"{fmt_bytes(ib[nm]['size']):>9} {fmt_bytes(pb+pc):>10}")
  528. del d
  529. if (i+1)%30==0: gc.collect()
  530.  
  531. if total_punched_p2 > 0:
  532. print(f"\n[PUNCH] Pass 2 freed: {fmt_bytes(total_punched_p2)}")
  533. print(f"[PUNCH] Disk free: {disk_free_gb():.1f} GB")
  534. else:
  535. print(f"\n[DELTA] Pass 2: compute + buffer {len(changed)} deltas...")
  536. print("[DELTA] WARNING: buffered mode — high RAM usage!")
  537. print(f"{'#':>4} {'Tensor':<52} {'Type':<10} {'Size':>9} {'Punched':>10}")
  538. print("-"*95)
  539. total_punched_p2 = 0
  540. for i,nm in enumerate(changed):
  541. ti = ib[nm]
  542. d = tensor_diff(fb, ti, fc, ic[nm], ti["qtype"])
  543. w.add_tensor(nm, d, raw_shape=np.array(ti["shape"],np.uint64),
  544. raw_dtype=ti["qtype"])
  545.  
  546. pb = pc = 0
  547. if _punch_hole_supported:
  548. pb = punch_hole(path_b, ib[nm]["offset"], ib[nm]["size"])
  549. pc = punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
  550. total_punched_p2 += pb + pc
  551.  
  552. print(f"{i+1:>4} {nm:<52} {ti['qtype_name']:<10} "
  553. f"{fmt_bytes(ti['size']):>9} {fmt_bytes(pb+pc):>10}")
  554. del d
  555. if (i+1)%30==0: gc.collect()
  556. w.write_header_to_file()
  557. w.write_kv_data_to_file()
  558. w.write_tensors_to_file()
  559.  
  560. if total_punched_p2 > 0:
  561. print(f"\n[PUNCH] Pass 2 freed: {fmt_bytes(total_punched_p2)}")
  562.  
  563. fb.close(); fc.close(); w.close()
  564. el = time.time()-t0
  565. print(f"\n[DELTA] Done: {len(changed)} tensors in delta GGUF "
  566. f"({file_size_gb(delta_path):.2f} GB, {el:.1f}s)")
  567. print(f"[DELTA] Disk free: {disk_free_gb():.1f} GB")
  568. return {"changed":len(changed),"unchanged":len(same)}
  569.  
  570. # ============================================================================
  571. # STAGE 2: APPLY DELTA TO A → output = A + delta * weight
  572. # A is RENAMED to output (no copy), then modified in-place
  573. # ============================================================================
  574.  
  575. def apply_delta(path_a, delta_path, weight, output_path):
  576. t0 = time.time()
  577. print("\n[APPLY] Parsing delta GGUF...")
  578. id_, nd = parse_gguf(delta_path)
  579. print(f"[APPLY] Delta: {nd} changed tensors")
  580.  
  581. if nd == 0:
  582. print("[APPLY] Empty delta → renaming A as output.")
  583. shutil.move(path_a, output_path); return {}
  584.  
  585. print("[APPLY] Parsing A...")
  586. ia, na = parse_gguf(path_a)
  587. sa = os.path.getsize(path_a)
  588. print(f"[APPLY] A: {na} tensors ({sa:,} B)")
  589.  
  590. miss = [n for n in id_ if n not in ia]
  591. if miss:
  592. print(f"[APPLY] WARNING: {len(miss)} delta tensors missing in A")
  593. for n in miss[:5]: print(f" {n}")
  594.  
  595. # ── In-place: rename A → output (no disk space needed) ───────────────
  596. abs_a = os.path.abspath(path_a)
  597. abs_o = os.path.abspath(output_path)
  598.  
  599. if abs_a == abs_o:
  600. print(f"[APPLY] A and output are the same file — modifying in-place.")
  601. else:
  602. print(f"[APPLY] Renaming A → {os.path.basename(output_path)} (no copy, 0 bytes needed)...")
  603. shutil.move(path_a, output_path)
  604. assert os.path.exists(output_path), "Rename failed"
  605. print(f"[APPLY] Rename OK — disk free: {disk_free_gb():.1f} GB")
  606.  
  607. fo = open(output_path, "r+b")
  608. fd = open(delta_path, "rb")
  609.  
  610. # Architecture info
  611. attn=set(); ssm=set(); ffn=set()
  612. for nm in ia:
  613. p = nm.split(".")
  614. if len(p)>=3 and p[0]=="blk":
  615. b = f"blk.{p[1]}"
  616. if ".attn_q." in nm: attn.add(b)
  617. if ".ssm_in." in nm: ssm.add(b)
  618. if ".ffn_gate." in nm: ffn.add(b)
  619. print(f"\n[APPLY] Arch: attn={len(attn)} deltanet={len(ssm)} ffn={len(ffn)}")
  620.  
  621. names = sorted(ia.keys())
  622. print(f"\n[APPLY] output = A + Δ * {weight} ({nd} deltas in {na} tensors)")
  623. print(f"{'#':>4} {'Tensor':<52} {'Type':<10} {'Size':>9} {'Action':<10} Detail")
  624. print("-"*125)
  625.  
  626. st = {"merged":0,"protected":0,"skipped":0,"total_changed":0,
  627. "identical":0,"failed":0,"attn":0,"ssm":0,"ffn":0}
  628.  
  629. for idx, nm in enumerate(names):
  630. ta = ia[nm]; qn = ta["qtype_name"]; sz = ta["size"]
  631. if nm not in id_:
  632. if is_protected(nm) or not is_finetune_weight(nm):
  633. st["protected"]+=1; lbl="PROTECT"
  634. rsn = "structural/norm/embd" if is_protected(nm) else "not fine-tunable"
  635. else:
  636. st["skipped"]+=1; lbl="KEEP"; rsn="unchanged (not in delta)"
  637. print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {lbl:<10} {rsn}")
  638. continue
  639.  
  640. td = id_[nm]
  641. if ta["qtype"] != td["qtype"]:
  642. st["skipped"]+=1
  643. print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'SKIP':<10} "
  644. f"qtype mismatch A={ta['qtype_name']} Δ={td['qtype_name']}"); continue
  645. if ta["size"] != td["size"]:
  646. st["skipped"]+=1
  647. print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'SKIP':<10} "
  648. f"size mismatch A={ta['size']} Δ={td['size']}"); continue
  649.  
  650. qt = ta["qtype"]; ds = ta["size"]
  651. ba = block_align(qt); cb = align_down(CHUNK_BYTES, ba)
  652. if cb<=0: cb=ba
  653.  
  654. chg=0; fail=0; off=0
  655. while off < ds:
  656. act = align_down(min(cb, ds-off), ba)
  657. if act<=0: break
  658. ra = read_chunk(fo, ta, off, act)
  659. rd = read_chunk(fd, td, off, act)
  660. if len(ra)!=act or len(rd)!=act: break
  661. try:
  662. fa = to_f32(ra, qt); fdd = to_f32(rd, qt)
  663. except Exception:
  664. fail+=1; del ra,rd; off+=act; continue
  665. n = min(len(fa), len(fdd))
  666. m = fa[:n] + fdd[:n] * np.float32(weight)
  667. del fa, fdd
  668. nr = from_f32(m, qt, len(ra)); del m
  669. if nr is not None:
  670. oa = np.frombuffer(ra, np.uint8)
  671. na_ = np.frombuffer(nr, np.uint8)
  672. chg += int(np.sum(oa != na_))
  673. write_chunk(fo, ta, off, nr)
  674. del oa, na_, nr
  675. else: fail+=1
  676. del ra, rd; off += act
  677.  
  678. st["merged"]+=1; st["total_changed"]+=chg; st["failed"]+=fail
  679. if ".attn_" in nm and "_norm" not in nm: st["attn"]+=1
  680. elif ".ssm_" in nm and "_norm" not in nm: st["ssm"]+=1
  681. elif ".ffn_" in nm and "_norm" not in nm: st["ffn"]+=1
  682.  
  683. if chg>0:
  684. pct=chg/sz*100; fs=f" ({fail} kept)" if fail else ""
  685. print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'MERGED':<10} "
  686. f"{chg:,}B ({pct:.2f}%){fs}")
  687. else:
  688. st["identical"]+=1
  689. print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'MERGED':<10} "
  690. f"(identical)")
  691. if (idx+1)%50==0: gc.collect()
  692.  
  693. flush_and_close(fo, "output"); fd.close()
  694. el = time.time()-t0
  695. print(f"\n{'='*75}\nAPPLY SUMMARY\n{'='*75}")
  696. print(f"Protected: {st['protected']}")
  697. print(f"Merged: {st['merged']} (attn={st['attn']} ssm={st['ssm']} ffn={st['ffn']})")
  698. print(f" Identical: {st['identical']}")
  699. print(f" Changed: {st['total_changed']:,} bytes")
  700. if st['failed']: print(f" Failed chk: {st['failed']}")
  701. print(f"Kept/Skipped: {st['skipped']}")
  702. print(f"Time: {el:.1f}s")
  703. print(f"Output: {os.path.basename(output_path)} ({file_size_gb(output_path):.2f} GB)")
  704. return st
  705.  
  706. # ============================================================================
  707. # VERIFY
  708. # ============================================================================
  709.  
  710. def verify(path):
  711. print("\n[VERIFY] Checking output...")
  712. try: info, n = parse_gguf(path); print(f"[VERIFY] Parsed {n} tensors.")
  713. except Exception as e: print(f"[VERIFY] ERROR: {e}"); return False
  714. f = open(path,"rb"); ok=w=0
  715. for nm in sorted(info.keys())[:30]:
  716. ti = info[nm]; cs = min(4096, ti["size"])
  717. raw = read_chunk(f, ti, 0, cs)
  718. a = np.frombuffer(raw, np.uint8)
  719. if a.sum()==0 and cs>64: w+=1; print(f" WARNING: '{nm}' all zeros!")
  720. else: ok+=1
  721. del raw, a
  722. f.close()
  723. if w: print(f"[VERIFY] {w} zero tensors.")
  724. print(f"[VERIFY] {ok} tensors OK."); return w==0
  725.  
  726. # ============================================================================
  727. # MAIN
  728. # ============================================================================
  729.  
  730. def main():
  731. print("="*75)
  732. print("GGUF ADD-DIFFERENCE MERGE — Disk-Optimized")
  733. print("output = A + (B - C) * weight")
  734. print()
  735. print("Pipeline:")
  736. print(" 1) Download B and C")
  737. print(" 2) Compute delta = B - C → GGUF (punch holes in B/C)")
  738. print(" 3) Delete B and C")
  739. print(" 4) Download A")
  740. print(" 5) Rename A → output, apply delta in-place (no copy)")
  741. print(" 6) Delete delta")
  742. print()
  743. print(f"WORK_DIR: {WORK_DIR}")
  744. print(f"OUTPUT: {OUTPUT_PATH}")
  745. print(f"DELTA: {DELTA_PATH}")
  746. print(f"WEIGHT: {MERGE_WEIGHT}")
  747. print(f"PUNCH_HOLE: {'YES' if _punch_hole_supported else 'NO'}")
  748. print("="*75)
  749.  
  750. # 1) Download B and C
  751. print("\n"+"="*75+"\nSTEP 1: Download B and C\n"+"="*75)
  752. pb = download(MODEL_B, "B (fine-tuned)")
  753. pc = download(MODEL_C, "C (reference)")
  754. print_workspace()
  755.  
  756. # 2) Compute delta
  757. print("\n"+"="*75+"\nSTEP 2: Compute delta = B - C → GGUF\n"+"="*75)
  758. compute_delta(pb, pc, DELTA_PATH)
  759. gc.collect(); print_workspace()
  760.  
  761. # 3) Delete B and C
  762. print("\n"+"="*75+"\nSTEP 3: Delete B and C\n"+"="*75)
  763. for p in set([os.path.abspath(pb), os.path.abspath(pc)]):
  764. safe_remove(p, protect=[OUTPUT_PATH, DELTA_PATH])
  765. gc.collect(); print_workspace()
  766.  
  767. # 4) Download A
  768. print("\n"+"="*75+"\nSTEP 4: Download A\n"+"="*75)
  769. pa = download(MODEL_A, "A (base)")
  770. print_workspace()
  771.  
  772. # 5) Apply delta (in-place, no copy)
  773. print("\n"+"="*75+"\nSTEP 5: Apply output = A + delta * weight (in-place)\n"+"="*75)
  774. apply_delta(pa, DELTA_PATH, MERGE_WEIGHT, OUTPUT_PATH)
  775. gc.collect()
  776.  
  777. # 6) Cleanup
  778. print("\n"+"="*75+"\nSTEP 6: Cleanup\n"+"="*75)
  779. # A was already renamed to output — only delete delta
  780. safe_remove(DELTA_PATH, protect=OUTPUT_PATH)
  781. gc.collect(); print_workspace()
  782.  
  783. # Verify
  784. print("\n"+"="*75+"\nVERIFY\n"+"="*75)
  785. verify(OUTPUT_PATH)
  786.  
  787. # Done
  788. print("\n"+"="*75+"\nDONE\n"+"="*75)
  789. print_workspace()
  790. print(f"\n Final: {OUTPUT_PATH}")
  791. print(f" Size: {file_size_gb(OUTPUT_PATH):.2f} GB")
  792. print(f" Disk free: {disk_free_gb():.1f} GB")
  793.  
  794. main()
Advertisement
Add Comment
Please, Sign In to add comment