LuffyTheFox

Untitled

Apr 14th, 2026
125
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 32.36 KB | None | 0 0
  1. # ============================================================================
  2. # GGUF Add-Difference Merge — Disk-Optimized
  3. #
  4. # output = A + (B - C) * weight
  5. #
  6. # Pipeline:
  7. # 1) Download B and C
  8. # 2) Compute delta = B - C → GGUF (only actually changed tensors)
  9. # 3) Delete B and C
  10. # 4) Download A
  11. # 5) Apply output = A + delta * weight (in-place rename, no copy)
  12. # 6) Delete delta (keep only merged output)
  13. #
  14. # • Delta is standard GGUF in original quantization format
  15. # • Only tensors where raw bytes B ≠ C are included
  16. # • Two-pass delta: pass1=raw byte compare, pass2=compute+write diffs
  17. # • Streaming GGUF write: one tensor in RAM at a time
  18. # • Punch-hole: processed tensors in B/C are deallocated via fallocate
  19. # • In-place apply: A is renamed to output, no disk copy needed
  20. # ============================================================================
  21.  
  22. import subprocess, sys, os, gc, time, shutil, ctypes, ctypes.util
  23. import numpy as np
  24.  
  25. WORK_DIR = "/content/gguf_workspace"
  26. os.makedirs(WORK_DIR, exist_ok=True)
  27.  
  28. def install():
  29. for pkg in ["huggingface_hub[hf_xet]", "gguf", "hf_xet"]:
  30. subprocess.check_call(
  31. [sys.executable, "-m", "pip", "install", "-q", pkg],
  32. stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
  33. )
  34. print("[OK] Dependencies installed.")
  35.  
  36. install()
  37.  
  38. from huggingface_hub import hf_hub_download
  39. from gguf import GGUFReader, GGUFWriter
  40. from gguf.constants import GGMLQuantizationType, GGML_QUANT_SIZES
  41. from gguf import quants as gguf_quants
  42.  
  43. # ── Enable hf_xet for downloads ──────────────────────────────────────────
  44. try:
  45. import hf_xet
  46. print("[OK] hf_xet backend loaded — Xet-accelerated downloads enabled.")
  47. except ImportError:
  48. print("[WARN] hf_xet not available — falling back to default download.")
  49.  
  50. # ── Punch-hole support (Linux fallocate) ─────────────────────────────────
  51. _FALLOC_FL_PUNCH_HOLE = 0x02
  52. _FALLOC_FL_KEEP_SIZE = 0x01
  53. _libc = None
  54. _punch_hole_supported = False
  55.  
  56. def _init_punch_hole():
  57. global _libc, _punch_hole_supported
  58. if sys.platform != "linux":
  59. print("[INFO] Punch-hole: not Linux — disabled.")
  60. return
  61. try:
  62. _libc_name = ctypes.util.find_library("c")
  63. if _libc_name:
  64. _libc = ctypes.CDLL(_libc_name, use_errno=True)
  65. tf = os.path.join(WORK_DIR, "_punch_test")
  66. with open(tf, "wb") as f:
  67. f.write(b"\xff" * 8192)
  68. fd = os.open(tf, os.O_RDWR)
  69. ret = _libc.fallocate(
  70. ctypes.c_int(fd),
  71. ctypes.c_int(_FALLOC_FL_PUNCH_HOLE | _FALLOC_FL_KEEP_SIZE),
  72. ctypes.c_longlong(0),
  73. ctypes.c_longlong(4096),
  74. )
  75. os.close(fd)
  76. os.remove(tf)
  77. if ret == 0:
  78. _punch_hole_supported = True
  79. print("[OK] Punch-hole (fallocate PUNCH_HOLE): supported.")
  80. else:
  81. print(f"[INFO] Punch-hole: fallocate returned {ret} — disabled.")
  82. else:
  83. print("[INFO] Punch-hole: libc not found — disabled.")
  84. except Exception as e:
  85. print(f"[INFO] Punch-hole: init failed ({e}) — disabled.")
  86.  
  87. _init_punch_hole()
  88.  
  89. def punch_hole(filepath, offset, length):
  90. if not _punch_hole_supported or _libc is None or length <= 0:
  91. return 0
  92. try:
  93. fd = os.open(filepath, os.O_RDWR)
  94. try:
  95. BS = 4096
  96. aligned_start = ((offset + BS - 1) // BS) * BS
  97. aligned_end = ((offset + length) // BS) * BS
  98. if aligned_end <= aligned_start:
  99. return 0
  100. punch_len = aligned_end - aligned_start
  101. ret = _libc.fallocate(
  102. ctypes.c_int(fd),
  103. ctypes.c_int(_FALLOC_FL_PUNCH_HOLE | _FALLOC_FL_KEEP_SIZE),
  104. ctypes.c_longlong(aligned_start),
  105. ctypes.c_longlong(punch_len),
  106. )
  107. if ret == 0:
  108. return punch_len
  109. return 0
  110. finally:
  111. os.close(fd)
  112. except Exception:
  113. return 0
  114.  
  115. def file_blocks_kb(filepath):
  116. try:
  117. st = os.stat(filepath)
  118. return (st.st_blocks * 512) / 1024
  119. except Exception:
  120. return -1
  121.  
  122. # ── Patch IQ block sizes ─────────────────────────────────────────────────
  123. _IQ = {
  124. "IQ1_S":(256,50),"IQ1_M":(256,56),"IQ2_XXS":(256,66),"IQ2_XS":(256,74),
  125. "IQ2_S":(256,82),"IQ3_XXS":(256,98),"IQ3_S":(256,110),
  126. "IQ4_NL":(32,18),"IQ4_XS":(256,136),
  127. }
  128. for _n,_g in _IQ.items():
  129. _qt = getattr(GGMLQuantizationType, _n, None)
  130. if _qt is not None and _qt not in GGML_QUANT_SIZES:
  131. GGML_QUANT_SIZES[_qt] = _g
  132.  
  133. # ── Probe quantize support ──────────────────────────────────────────────
  134. _QUANTIZE_SUPPORTED = set()
  135.  
  136. def _probe_quantize():
  137. for qt in GGMLQuantizationType:
  138. if qt in (GGMLQuantizationType.F32, GGMLQuantizationType.F16,
  139. GGMLQuantizationType.BF16):
  140. _QUANTIZE_SUPPORTED.add(qt)
  141. elif qt in GGML_QUANT_SIZES:
  142. be, _ = GGML_QUANT_SIZES[qt]
  143. try:
  144. r = gguf_quants.quantize(np.zeros(be, np.float32), qt)
  145. if r is not None and len(r.ravel().tobytes()) > 0:
  146. _QUANTIZE_SUPPORTED.add(qt)
  147. except Exception: pass
  148.  
  149. _probe_quantize()
  150. print(f"[INFO] quantize() supported: {sorted([q.name for q in _QUANTIZE_SUPPORTED])}")
  151.  
  152. # ── Check streaming GGUF write ──────────────────────────────────────────
  153. _STREAMING = (
  154. hasattr(GGUFWriter, 'add_tensor_info') and
  155. hasattr(GGUFWriter, 'write_tensor_data') and
  156. hasattr(GGUFWriter, 'write_ti_data_to_file')
  157. )
  158. print(f"[INFO] GGUF streaming write: {'YES' if _STREAMING else 'NO (buffered)'}")
  159.  
  160. # ============================================================================
  161. # CONFIGURATION
  162. # ============================================================================
  163. MODEL_A = {
  164. "repo": "LuffyTheFox/Qwopus3.5-27B-v3-Uncensored-FernflowerAI-KL-ReLU-GGUF",
  165. "file": "Qwopus3.5-27B-v3-Uncensored-FernflowerAI-KL-ReLU.Q8_0.gguf",
  166. }
  167. MODEL_B = {
  168. "repo": "LuffyTheFox/FernflowerAI-27B-Uncensored-RYS-KL-ReLU-GGUF",
  169. "file": "FernflowerAI-27B-Uncensored-RYS-KL-ReLU-Q8_0.gguf",
  170. }
  171. MODEL_C = {
  172. "repo": "LuffyTheFox/Qwen3.5-27B-Uncensored-FernflowerAI-KL-ReLU-GGUF",
  173. "file": "FernflowerAI-27B-Uncensored-KL-ReLU.Q8_0.gguf",
  174. }
  175.  
  176. MERGE_WEIGHT = 1.0
  177. OUTPUT_NAME = "Qwopus-3.5-27B-RYS-Uncensored-KL.Q8_0.gguf"
  178. OUTPUT_PATH = os.path.join(WORK_DIR, OUTPUT_NAME)
  179. DELTA_PATH = os.path.join(WORK_DIR, "_delta.gguf")
  180. CHUNK_BYTES = 4 * 1024 * 1024
  181.  
  182. # ============================================================================
  183. # TENSOR CLASSIFICATION
  184. # ============================================================================
  185. FINETUNE_SUFFIXES = (
  186. "attn_q.weight","attn_k.weight","attn_v.weight","attn_output.weight",
  187. "ssm_in.weight","ssm_out.weight","ssm_a.weight",
  188. "ssm_d.weight","ssm_dt.weight",
  189. "ffn_gate.weight","ffn_up.weight","ffn_down.weight",
  190. "ffn_gate_exps.weight","ffn_up_exps.weight","ffn_down_exps.weight",
  191. "ffn_gate_inp.weight","ffn_gate_shexp.weight",
  192. "ffn_up_shexp.weight","ffn_down_shexp.weight",
  193. "attn_gate.weight",
  194. )
  195.  
  196. PROTECTED_SUBSTRINGS = (
  197. "token_embd","output.weight","_norm",".bias",
  198. "rope_freqs","pos_embd",
  199. "v.blk.","v.patch_embd","v.position_embd",
  200. "v.post_ln","v.pre_ln","v.output_norm","v.proj",
  201. "mm.","mtp.",
  202. )
  203.  
  204. FLOAT_QTYPES = {
  205. GGMLQuantizationType.F32, GGMLQuantizationType.F16, GGMLQuantizationType.BF16,
  206. }
  207.  
  208. # ============================================================================
  209. # UTILITIES
  210. # ============================================================================
  211.  
  212. def disk_free_gb(p=WORK_DIR):
  213. st = os.statvfs(p); return (st.f_bavail * st.f_frsize) / 1073741824
  214.  
  215. def file_size_gb(p):
  216. return os.path.getsize(p) / 1073741824 if os.path.exists(p) else 0.0
  217.  
  218. def fmt_bytes(n):
  219. for u in ("B","KB","MB","GB"):
  220. if n < 1024: return f"{n:.1f} {u}"
  221. n /= 1024
  222. return f"{n:.1f} TB"
  223.  
  224. def safe_remove(path, protect=None):
  225. try:
  226. a = os.path.abspath(path)
  227. if protect:
  228. if isinstance(protect, str): protect = [protect]
  229. for p in protect:
  230. if a == os.path.abspath(p): return
  231. if os.path.isfile(a):
  232. sz = file_size_gb(a)
  233. os.remove(a)
  234. print(f"[CLEANUP] Removed {os.path.basename(a)} ({sz:.2f} GB)")
  235. except OSError: pass
  236.  
  237. def flush_and_close(f, label="file"):
  238. f.flush(); os.fsync(f.fileno()); f.close()
  239. print(f"[FLUSH] {label}: flush + fsync done.")
  240.  
  241. def block_align(qt):
  242. if qt in FLOAT_QTYPES:
  243. return {GGMLQuantizationType.F32:4,GGMLQuantizationType.F16:2,
  244. GGMLQuantizationType.BF16:2}[qt]
  245. if qt in GGML_QUANT_SIZES: return GGML_QUANT_SIZES[qt][1]
  246. return 1
  247.  
  248. def align_down(v, a):
  249. return (v // a) * a if a > 0 else v
  250.  
  251. def print_workspace():
  252. print(f"\n[FS] {WORK_DIR}:")
  253. for fn in sorted(os.listdir(WORK_DIR)):
  254. fp = os.path.join(WORK_DIR, fn)
  255. if os.path.isfile(fp):
  256. logical = file_size_gb(fp)
  257. actual_kb = file_blocks_kb(fp)
  258. if actual_kb >= 0:
  259. actual_gb = actual_kb / (1024*1024)
  260. saved = logical - actual_gb
  261. extra = f" (on-disk: {actual_gb:.2f} GB, saved: {saved:.2f} GB)" if saved > 0.01 else ""
  262. else:
  263. extra = ""
  264. print(f" {fn} ({logical:.2f} GB){extra}")
  265. print(f" Disk free: {disk_free_gb():.1f} GB")
  266.  
  267. # ============================================================================
  268. # CLASSIFICATION
  269. # ============================================================================
  270.  
  271. def is_finetune_weight(name):
  272. return any(name.endswith(s) for s in FINETUNE_SUFFIXES)
  273.  
  274. def is_protected(name):
  275. return any(p in name for p in PROTECTED_SUBSTRINGS)
  276.  
  277. def classify_bc(name, ti_b, ti_c):
  278. if is_protected(name): return "protect", "structural/norm/embd"
  279. if not is_finetune_weight(name): return "protect", "not fine-tunable"
  280. if ti_c is None: return "skip", "missing in C"
  281. if ti_b["size"] != ti_c["size"]:
  282. return "skip", f"size mismatch B={ti_b['size']:,} C={ti_c['size']:,}"
  283. if ti_b["qtype_name"] != ti_c["qtype_name"]:
  284. return "skip", f"type mismatch B={ti_b['qtype_name']} C={ti_c['qtype_name']}"
  285. if ti_b["n_elements"] != ti_c["n_elements"]:
  286. return "skip", f"elem mismatch"
  287. if ti_b["qtype"] not in _QUANTIZE_SUPPORTED:
  288. return "skip", f"{ti_b['qtype_name']} no quantize()"
  289. return "merge", "fine-tunable"
  290.  
  291. # ============================================================================
  292. # GGUF PARSING
  293. # ============================================================================
  294.  
  295. def parse_gguf(path):
  296. reader = GGUFReader(path)
  297. tensors = {}
  298. for t in reader.tensors:
  299. tensors[t.name] = {
  300. "offset": int(t.data_offset), "size": int(t.n_bytes),
  301. "qtype": t.tensor_type, "qtype_name": t.tensor_type.name,
  302. "n_elements": int(t.n_elements), "shape": list(t.shape),
  303. }
  304. n = len(tensors); del reader; gc.collect()
  305. return tensors, n
  306.  
  307. # ============================================================================
  308. # RAW I/O
  309. # ============================================================================
  310.  
  311. def read_chunk(f, ti, off, sz):
  312. actual = min(sz, ti["size"] - off)
  313. if actual <= 0: return b""
  314. f.seek(ti["offset"] + off); return f.read(actual)
  315.  
  316. def write_chunk(f, ti, off, data):
  317. f.seek(ti["offset"] + off); f.write(data)
  318.  
  319. def raw_equal(f1, t1, f2, t2):
  320. if t1["size"] != t2["size"]: return False
  321. sz = t1["size"]; off = 0
  322. while off < sz:
  323. n = min(CHUNK_BYTES, sz - off)
  324. f1.seek(t1["offset"]+off); d1 = f1.read(n)
  325. f2.seek(t2["offset"]+off); d2 = f2.read(n)
  326. if d1 != d2: return False
  327. off += n
  328. return True
  329.  
  330. # ============================================================================
  331. # DEQUANT / QUANT
  332. # ============================================================================
  333.  
  334. def to_f32(raw, qt):
  335. if qt == GGMLQuantizationType.F32:
  336. return np.frombuffer(raw, np.float32).copy()
  337. if qt == GGMLQuantizationType.F16:
  338. return np.frombuffer(raw, np.float16).astype(np.float32)
  339. if qt == GGMLQuantizationType.BF16:
  340. u = np.frombuffer(raw, np.uint16).copy()
  341. return (u.astype(np.uint32) << 16).view(np.float32).copy()
  342. return gguf_quants.dequantize(np.frombuffer(raw, np.uint8).copy(), qt
  343. ).ravel().astype(np.float32)
  344.  
  345. def from_f32(f, qt, expected):
  346. f = np.nan_to_num(f, nan=0., posinf=0., neginf=0.)
  347. if qt == GGMLQuantizationType.F32:
  348. r = f.astype(np.float32).tobytes()
  349. elif qt == GGMLQuantizationType.F16:
  350. r = f.astype(np.float16).tobytes()
  351. elif qt == GGMLQuantizationType.BF16:
  352. r = (f.astype(np.float32).view(np.uint32) >> 16).astype(np.uint16).tobytes()
  353. elif qt in _QUANTIZE_SUPPORTED:
  354. be, _ = GGML_QUANT_SIZES[qt]
  355. n = align_down(len(f), be)
  356. if n == 0: return None
  357. try: r = gguf_quants.quantize(f[:n], qt).ravel().tobytes()
  358. except Exception: return None
  359. else: return None
  360. return r if len(r) == expected else None
  361.  
  362. def quant_nd(f, qt):
  363. f = np.nan_to_num(f, nan=0., posinf=0., neginf=0.).astype(np.float32)
  364. if qt == GGMLQuantizationType.F32: return f.view(np.uint8).copy()
  365. if qt == GGMLQuantizationType.F16: return f.astype(np.float16).view(np.uint8).copy()
  366. if qt == GGMLQuantizationType.BF16:
  367. return (f.view(np.uint32) >> 16).astype(np.uint16).view(np.uint8).copy()
  368. if qt in _QUANTIZE_SUPPORTED:
  369. be, _ = GGML_QUANT_SIZES[qt]; n = align_down(len(f), be)
  370. if n == 0: return np.zeros(0, np.uint8)
  371. return gguf_quants.quantize(f[:n], qt).ravel().copy()
  372. return np.zeros(0, np.uint8)
  373.  
  374. # ============================================================================
  375. # DOWNLOAD
  376. # ============================================================================
  377.  
  378. def download(spec, label):
  379. local_path = os.path.join(WORK_DIR, spec["file"])
  380.  
  381. # Skip download if the file already exists in WORK_DIR
  382. if os.path.isfile(local_path) and os.path.getsize(local_path) > 0:
  383. print(f"[DL] {label}: already exists — skipping download.")
  384. print(f"[DL] {label}: {file_size_gb(local_path):.2f} GB -> {local_path}")
  385. return local_path
  386.  
  387. print(f"[DL] {label}: {spec['repo']} / {spec['file']}")
  388. p = hf_hub_download(spec["repo"], spec["file"], local_dir=WORK_DIR)
  389. print(f"[DL] {label}: {file_size_gb(p):.2f} GB -> {p}"); return p
  390.  
  391. # ============================================================================
  392. # COMPUTE ONE TENSOR DIFF → quantized uint8 ndarray
  393. # ============================================================================
  394.  
  395. def tensor_diff(f_b, tb, f_c, tc, qt):
  396. sz = tb["size"]; ba = block_align(qt)
  397. cb = align_down(CHUNK_BYTES, ba)
  398. if cb <= 0: cb = ba
  399. parts = []; off = 0
  400. while off < sz:
  401. act = align_down(min(cb, sz - off), ba)
  402. if act <= 0: break
  403. rb = read_chunk(f_b, tb, off, act)
  404. rc = read_chunk(f_c, tc, off, act)
  405. if len(rb) != act or len(rc) != act: break
  406. try:
  407. d = to_f32(rb, qt) - to_f32(rc, qt)
  408. parts.append(np.nan_to_num(d, nan=0., posinf=0., neginf=0.))
  409. except Exception:
  410. if qt in FLOAT_QTYPES:
  411. eb = {GGMLQuantizationType.F32:4,GGMLQuantizationType.F16:2,
  412. GGMLQuantizationType.BF16:2}[qt]
  413. parts.append(np.zeros(act//eb, np.float32))
  414. elif qt in GGML_QUANT_SIZES:
  415. be, bs = GGML_QUANT_SIZES[qt]
  416. parts.append(np.zeros((act//bs)*be, np.float32))
  417. del rb, rc; off += act
  418. full = np.concatenate(parts) if parts else np.zeros(tb["n_elements"], np.float32)
  419. del parts; r = quant_nd(full, qt); del full; return r
  420.  
  421. # ============================================================================
  422. # STAGE 1: COMPUTE DELTA = B - C → GGUF (only changed tensors)
  423. # + punch holes in B and C after each tensor is processed
  424. # ============================================================================
  425.  
  426. def compute_delta(path_b, path_c, delta_path):
  427. t0 = time.time()
  428. print("\n[DELTA] Parsing B and C...")
  429. ib, nb = parse_gguf(path_b); ic, nc = parse_gguf(path_c)
  430. print(f"[DELTA] B: {nb} tensors C: {nc} tensors")
  431.  
  432. plan = {}; mergeable = []
  433. for nm in sorted(ib.keys()):
  434. a, r = classify_bc(nm, ib[nm], ic.get(nm))
  435. plan[nm] = (a, r)
  436. if a == "merge": mergeable.append(nm)
  437.  
  438. np_ = sum(1 for v in plan.values() if v[0]=="protect")
  439. ns = sum(1 for v in plan.values() if v[0]=="skip")
  440. print(f"[DELTA] Mergeable={len(mergeable)} Protected={np_} Skipped={ns}")
  441. for nm,(a,r) in sorted(plan.items()):
  442. if a=="skip": print(f" SKIP: {nm:<48} {r}")
  443.  
  444. # ── Punch holes for protected/skipped tensors immediately ────────────
  445. if _punch_hole_supported:
  446. total_punched = 0
  447. for nm in sorted(ib.keys()):
  448. act, _ = plan.get(nm, ("protect",""))
  449. if act != "merge":
  450. tb = ib[nm]
  451. p = punch_hole(path_b, tb["offset"], tb["size"])
  452. total_punched += p
  453. tc = ic.get(nm)
  454. if tc is not None:
  455. p = punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
  456. total_punched += p
  457. if total_punched > 0:
  458. print(f"[PUNCH] Freed {fmt_bytes(total_punched)} from "
  459. f"protected/skipped tensors in B and C")
  460. print(f"[PUNCH] Disk free: {disk_free_gb():.1f} GB")
  461.  
  462. if not mergeable:
  463. w = GGUFWriter(delta_path, arch="delta")
  464. w.add_description("Empty delta")
  465. w.write_header_to_file(); w.write_kv_data_to_file()
  466. w.write_ti_data_to_file(); w.close()
  467. print("[DELTA] Empty delta created."); return {}
  468.  
  469. # ── Pass 1: raw byte compare ─────────────────────────────────────────
  470. print(f"\n[DELTA] Pass 1: raw byte compare ({len(mergeable)} tensors)...")
  471. fb = open(path_b,"rb"); fc = open(path_c,"rb")
  472. changed=[]; same=[]
  473. for i,nm in enumerate(mergeable):
  474. eq = raw_equal(fb, ib[nm], fc, ic[nm])
  475. if eq:
  476. same.append(nm)
  477. if _punch_hole_supported:
  478. punch_hole(path_b, ib[nm]["offset"], ib[nm]["size"])
  479. punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
  480. else:
  481. changed.append(nm)
  482. if (i+1)%100==0 or i+1==len(mergeable):
  483. print(f" {i+1}/{len(mergeable)} changed={len(changed)} same={len(same)}"
  484. f" disk_free={disk_free_gb():.1f}GB")
  485.  
  486. fb.close(); fc.close()
  487.  
  488. print(f"[DELTA] Result: {len(changed)} changed, {len(same)} identical")
  489. if _punch_hole_supported and same:
  490. print(f"[PUNCH] Punched {len(same)} identical tensors from B and C")
  491. print(f"[PUNCH] Disk free: {disk_free_gb():.1f} GB")
  492.  
  493. if not changed:
  494. w = GGUFWriter(delta_path, arch="delta")
  495. w.add_description("Empty delta: B==C")
  496. w.write_header_to_file(); w.write_kv_data_to_file()
  497. w.write_ti_data_to_file(); w.close()
  498. print("[DELTA] Empty delta."); return {"changed":0,"unchanged":len(same)}
  499.  
  500. # ── Write delta GGUF ─────────────────────────────────────────────────
  501. w = GGUFWriter(delta_path, arch="delta")
  502. w.add_description(f"Delta (B-C): {len(changed)} changed tensors. "
  503. f"B={MODEL_B['repo']}/{MODEL_B['file']} "
  504. f"C={MODEL_C['repo']}/{MODEL_C['file']}")
  505. w.add_uint32("delta.changed_count", len(changed))
  506. w.add_uint32("delta.unchanged_count", len(same))
  507.  
  508. fb = open(path_b,"rb"); fc = open(path_c,"rb")
  509.  
  510. if _STREAMING:
  511. for nm in changed:
  512. ti = ib[nm]
  513. w.add_tensor_info(name=nm, tensor_shape=ti["shape"],
  514. tensor_dtype=np.float32,
  515. tensor_nbytes=ti["size"], raw_dtype=ti["qtype"])
  516. w.write_header_to_file()
  517. w.write_kv_data_to_file()
  518. w.write_ti_data_to_file()
  519.  
  520. print(f"\n[DELTA] Pass 2: compute + stream {len(changed)} deltas...")
  521. print(f"{'#':>4} {'Tensor':<52} {'Type':<10} {'Size':>9} {'Punched':>10}")
  522. print("-"*95)
  523. total_punched_p2 = 0
  524. for i,nm in enumerate(changed):
  525. d = tensor_diff(fb, ib[nm], fc, ic[nm], ib[nm]["qtype"])
  526. w.write_tensor_data(d)
  527.  
  528. pb = pc = 0
  529. if _punch_hole_supported:
  530. pb = punch_hole(path_b, ib[nm]["offset"], ib[nm]["size"])
  531. pc = punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
  532. total_punched_p2 += pb + pc
  533.  
  534. print(f"{i+1:>4} {nm:<52} {ib[nm]['qtype_name']:<10} "
  535. f"{fmt_bytes(ib[nm]['size']):>9} {fmt_bytes(pb+pc):>10}")
  536. del d
  537. if (i+1)%30==0: gc.collect()
  538.  
  539. if total_punched_p2 > 0:
  540. print(f"\n[PUNCH] Pass 2 freed: {fmt_bytes(total_punched_p2)}")
  541. print(f"[PUNCH] Disk free: {disk_free_gb():.1f} GB")
  542. else:
  543. print(f"\n[DELTA] Pass 2: compute + buffer {len(changed)} deltas...")
  544. print("[DELTA] WARNING: buffered mode — high RAM usage!")
  545. print(f"{'#':>4} {'Tensor':<52} {'Type':<10} {'Size':>9} {'Punched':>10}")
  546. print("-"*95)
  547. total_punched_p2 = 0
  548. for i,nm in enumerate(changed):
  549. ti = ib[nm]
  550. d = tensor_diff(fb, ti, fc, ic[nm], ti["qtype"])
  551. w.add_tensor(nm, d, raw_shape=np.array(ti["shape"],np.uint64),
  552. raw_dtype=ti["qtype"])
  553.  
  554. pb = pc = 0
  555. if _punch_hole_supported:
  556. pb = punch_hole(path_b, ib[nm]["offset"], ib[nm]["size"])
  557. pc = punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
  558. total_punched_p2 += pb + pc
  559.  
  560. print(f"{i+1:>4} {nm:<52} {ti['qtype_name']:<10} "
  561. f"{fmt_bytes(ti['size']):>9} {fmt_bytes(pb+pc):>10}")
  562. del d
  563. if (i+1)%30==0: gc.collect()
  564. w.write_header_to_file()
  565. w.write_kv_data_to_file()
  566. w.write_tensors_to_file()
  567.  
  568. if total_punched_p2 > 0:
  569. print(f"\n[PUNCH] Pass 2 freed: {fmt_bytes(total_punched_p2)}")
  570.  
  571. fb.close(); fc.close(); w.close()
  572. el = time.time()-t0
  573. print(f"\n[DELTA] Done: {len(changed)} tensors in delta GGUF "
  574. f"({file_size_gb(delta_path):.2f} GB, {el:.1f}s)")
  575. print(f"[DELTA] Disk free: {disk_free_gb():.1f} GB")
  576. return {"changed":len(changed),"unchanged":len(same)}
  577.  
  578. # ============================================================================
  579. # STAGE 2: APPLY DELTA TO A → output = A + delta * weight
  580. # A is RENAMED to output (no copy), then modified in-place
  581. # ============================================================================
  582.  
  583. def apply_delta(path_a, delta_path, weight, output_path):
  584. t0 = time.time()
  585. print("\n[APPLY] Parsing delta GGUF...")
  586. id_, nd = parse_gguf(delta_path)
  587. print(f"[APPLY] Delta: {nd} changed tensors")
  588.  
  589. if nd == 0:
  590. print("[APPLY] Empty delta → renaming A as output.")
  591. shutil.move(path_a, output_path); return {}
  592.  
  593. print("[APPLY] Parsing A...")
  594. ia, na = parse_gguf(path_a)
  595. sa = os.path.getsize(path_a)
  596. print(f"[APPLY] A: {na} tensors ({sa:,} B)")
  597.  
  598. miss = [n for n in id_ if n not in ia]
  599. if miss:
  600. print(f"[APPLY] WARNING: {len(miss)} delta tensors missing in A")
  601. for n in miss[:5]: print(f" {n}")
  602.  
  603. # ── In-place: rename A → output (no disk space needed) ───────────────
  604. abs_a = os.path.abspath(path_a)
  605. abs_o = os.path.abspath(output_path)
  606.  
  607. if abs_a == abs_o:
  608. print(f"[APPLY] A and output are the same file — modifying in-place.")
  609. else:
  610. print(f"[APPLY] Renaming A → {os.path.basename(output_path)} (no copy, 0 bytes needed)...")
  611. shutil.move(path_a, output_path)
  612. assert os.path.exists(output_path), "Rename failed"
  613. print(f"[APPLY] Rename OK — disk free: {disk_free_gb():.1f} GB")
  614.  
  615. fo = open(output_path, "r+b")
  616. fd = open(delta_path, "rb")
  617.  
  618. # Architecture info
  619. attn=set(); ssm=set(); ffn=set()
  620. for nm in ia:
  621. p = nm.split(".")
  622. if len(p)>=3 and p[0]=="blk":
  623. b = f"blk.{p[1]}"
  624. if ".attn_q." in nm: attn.add(b)
  625. if ".ssm_in." in nm: ssm.add(b)
  626. if ".ffn_gate." in nm: ffn.add(b)
  627. print(f"\n[APPLY] Arch: attn={len(attn)} deltanet={len(ssm)} ffn={len(ffn)}")
  628.  
  629. names = sorted(ia.keys())
  630. print(f"\n[APPLY] output = A + Δ * {weight} ({nd} deltas in {na} tensors)")
  631. print(f"{'#':>4} {'Tensor':<52} {'Type':<10} {'Size':>9} {'Action':<10} Detail")
  632. print("-"*125)
  633.  
  634. st = {"merged":0,"protected":0,"skipped":0,"total_changed":0,
  635. "identical":0,"failed":0,"attn":0,"ssm":0,"ffn":0}
  636.  
  637. for idx, nm in enumerate(names):
  638. ta = ia[nm]; qn = ta["qtype_name"]; sz = ta["size"]
  639. if nm not in id_:
  640. if is_protected(nm) or not is_finetune_weight(nm):
  641. st["protected"]+=1; lbl="PROTECT"
  642. rsn = "structural/norm/embd" if is_protected(nm) else "not fine-tunable"
  643. else:
  644. st["skipped"]+=1; lbl="KEEP"; rsn="unchanged (not in delta)"
  645. print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {lbl:<10} {rsn}")
  646. continue
  647.  
  648. td = id_[nm]
  649. if ta["qtype"] != td["qtype"]:
  650. st["skipped"]+=1
  651. print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'SKIP':<10} "
  652. f"qtype mismatch A={ta['qtype_name']} Δ={td['qtype_name']}"); continue
  653. if ta["size"] != td["size"]:
  654. st["skipped"]+=1
  655. print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'SKIP':<10} "
  656. f"size mismatch A={ta['size']} Δ={td['size']}"); continue
  657.  
  658. qt = ta["qtype"]; ds = ta["size"]
  659. ba = block_align(qt); cb = align_down(CHUNK_BYTES, ba)
  660. if cb<=0: cb=ba
  661.  
  662. chg=0; fail=0; off=0
  663. while off < ds:
  664. act = align_down(min(cb, ds-off), ba)
  665. if act<=0: break
  666. ra = read_chunk(fo, ta, off, act)
  667. rd = read_chunk(fd, td, off, act)
  668. if len(ra)!=act or len(rd)!=act: break
  669. try:
  670. fa = to_f32(ra, qt); fdd = to_f32(rd, qt)
  671. except Exception:
  672. fail+=1; del ra,rd; off+=act; continue
  673. n = min(len(fa), len(fdd))
  674. m = fa[:n] + fdd[:n] * np.float32(weight)
  675. del fa, fdd
  676. nr = from_f32(m, qt, len(ra)); del m
  677. if nr is not None:
  678. oa = np.frombuffer(ra, np.uint8)
  679. na_ = np.frombuffer(nr, np.uint8)
  680. chg += int(np.sum(oa != na_))
  681. write_chunk(fo, ta, off, nr)
  682. del oa, na_, nr
  683. else: fail+=1
  684. del ra, rd; off += act
  685.  
  686. st["merged"]+=1; st["total_changed"]+=chg; st["failed"]+=fail
  687. if ".attn_" in nm and "_norm" not in nm: st["attn"]+=1
  688. elif ".ssm_" in nm and "_norm" not in nm: st["ssm"]+=1
  689. elif ".ffn_" in nm and "_norm" not in nm: st["ffn"]+=1
  690.  
  691. if chg>0:
  692. pct=chg/sz*100; fs=f" ({fail} kept)" if fail else ""
  693. print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'MERGED':<10} "
  694. f"{chg:,}B ({pct:.2f}%){fs}")
  695. else:
  696. st["identical"]+=1
  697. print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'MERGED':<10} "
  698. f"(identical)")
  699. if (idx+1)%50==0: gc.collect()
  700.  
  701. flush_and_close(fo, "output"); fd.close()
  702. el = time.time()-t0
  703. print(f"\n{'='*75}\nAPPLY SUMMARY\n{'='*75}")
  704. print(f"Protected: {st['protected']}")
  705. print(f"Merged: {st['merged']} (attn={st['attn']} ssm={st['ssm']} ffn={st['ffn']})")
  706. print(f" Identical: {st['identical']}")
  707. print(f" Changed: {st['total_changed']:,} bytes")
  708. if st['failed']: print(f" Failed chk: {st['failed']}")
  709. print(f"Kept/Skipped: {st['skipped']}")
  710. print(f"Time: {el:.1f}s")
  711. print(f"Output: {os.path.basename(output_path)} ({file_size_gb(output_path):.2f} GB)")
  712. return st
  713.  
  714. # ============================================================================
  715. # VERIFY
  716. # ============================================================================
  717.  
  718. def verify(path):
  719. print("\n[VERIFY] Checking output...")
  720. try: info, n = parse_gguf(path); print(f"[VERIFY] Parsed {n} tensors.")
  721. except Exception as e: print(f"[VERIFY] ERROR: {e}"); return False
  722. f = open(path,"rb"); ok=w=0
  723. for nm in sorted(info.keys())[:30]:
  724. ti = info[nm]; cs = min(4096, ti["size"])
  725. raw = read_chunk(f, ti, 0, cs)
  726. a = np.frombuffer(raw, np.uint8)
  727. if a.sum()==0 and cs>64: w+=1; print(f" WARNING: '{nm}' all zeros!")
  728. else: ok+=1
  729. del raw, a
  730. f.close()
  731. if w: print(f"[VERIFY] {w} zero tensors.")
  732. print(f"[VERIFY] {ok} tensors OK."); return w==0
  733.  
  734. # ============================================================================
  735. # MAIN
  736. # ============================================================================
  737.  
  738. def main():
  739. print("="*75)
  740. print("GGUF ADD-DIFFERENCE MERGE — Disk-Optimized")
  741. print("output = A + (B - C) * weight")
  742. print()
  743. print("Pipeline:")
  744. print(" 1) Download B and C")
  745. print(" 2) Compute delta = B - C → GGUF (punch holes in B/C)")
  746. print(" 3) Delete B and C")
  747. print(" 4) Download A")
  748. print(" 5) Rename A → output, apply delta in-place (no copy)")
  749. print(" 6) Delete delta")
  750. print()
  751. print(f"WORK_DIR: {WORK_DIR}")
  752. print(f"OUTPUT: {OUTPUT_PATH}")
  753. print(f"DELTA: {DELTA_PATH}")
  754. print(f"WEIGHT: {MERGE_WEIGHT}")
  755. print(f"PUNCH_HOLE: {'YES' if _punch_hole_supported else 'NO'}")
  756. print("="*75)
  757.  
  758. # 1) Download B and C
  759. print("\n"+"="*75+"\nSTEP 1: Download B and C\n"+"="*75)
  760. pb = download(MODEL_B, "B (fine-tuned)")
  761. pc = download(MODEL_C, "C (reference)")
  762. print_workspace()
  763.  
  764. # 2) Compute delta
  765. print("\n"+"="*75+"\nSTEP 2: Compute delta = B - C → GGUF\n"+"="*75)
  766. compute_delta(pb, pc, DELTA_PATH)
  767. gc.collect(); print_workspace()
  768.  
  769. # 3) Delete B and C
  770. print("\n"+"="*75+"\nSTEP 3: Delete B and C\n"+"="*75)
  771. for p in set([os.path.abspath(pb), os.path.abspath(pc)]):
  772. safe_remove(p, protect=[OUTPUT_PATH, DELTA_PATH])
  773. gc.collect(); print_workspace()
  774.  
  775. # 4) Download A
  776. print("\n"+"="*75+"\nSTEP 4: Download A\n"+"="*75)
  777. pa = download(MODEL_A, "A (base)")
  778. print_workspace()
  779.  
  780. # 5) Apply delta (in-place, no copy)
  781. print("\n"+"="*75+"\nSTEP 5: Apply output = A + delta * weight (in-place)\n"+"="*75)
  782. apply_delta(pa, DELTA_PATH, MERGE_WEIGHT, OUTPUT_PATH)
  783. gc.collect()
  784.  
  785. # 6) Cleanup
  786. print("\n"+"="*75+"\nSTEP 6: Cleanup\n"+"="*75)
  787. # A was already renamed to output — only delete delta
  788. safe_remove(DELTA_PATH, protect=OUTPUT_PATH)
  789. gc.collect(); print_workspace()
  790.  
  791. # Verify
  792. print("\n"+"="*75+"\nVERIFY\n"+"="*75)
  793. verify(OUTPUT_PATH)
  794.  
  795. # Done
  796. print("\n"+"="*75+"\nDONE\n"+"="*75)
  797. print_workspace()
  798. print(f"\n Final: {OUTPUT_PATH}")
  799. print(f" Size: {file_size_gb(OUTPUT_PATH):.2f} GB")
  800. print(f" Disk free: {disk_free_gb():.1f} GB")
  801.  
  802. main()
Advertisement
Add Comment
Please, Sign In to add comment