Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # ============================================================================
- # GGUF Add-Difference Merge — Disk-Optimized
- #
- # output = A + (B - C) * weight
- #
- # Pipeline:
- # 1) Download B and C
- # 2) Compute delta = B - C → GGUF (only actually changed tensors)
- # 3) Delete B and C
- # 4) Download A
- # 5) Apply output = A + delta * weight (in-place rename, no copy)
- # 6) Delete delta (keep only merged output)
- #
- # • Delta is standard GGUF in original quantization format
- # • Only tensors where raw bytes B ≠ C are included
- # • Two-pass delta: pass1=raw byte compare, pass2=compute+write diffs
- # • Streaming GGUF write: one tensor in RAM at a time
- # • Punch-hole: processed tensors in B/C are deallocated via fallocate
- # • In-place apply: A is renamed to output, no disk copy needed
- # ============================================================================
- import subprocess, sys, os, gc, time, shutil, ctypes, ctypes.util
- import numpy as np
- WORK_DIR = "/content/gguf_workspace"
- os.makedirs(WORK_DIR, exist_ok=True)
- def install():
- for pkg in ["huggingface_hub[hf_xet]", "gguf", "hf_xet"]:
- subprocess.check_call(
- [sys.executable, "-m", "pip", "install", "-q", pkg],
- stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
- )
- print("[OK] Dependencies installed.")
- install()
- from huggingface_hub import hf_hub_download
- from gguf import GGUFReader, GGUFWriter
- from gguf.constants import GGMLQuantizationType, GGML_QUANT_SIZES
- from gguf import quants as gguf_quants
- # ── Enable hf_xet for downloads ──────────────────────────────────────────
- try:
- import hf_xet
- print("[OK] hf_xet backend loaded — Xet-accelerated downloads enabled.")
- except ImportError:
- print("[WARN] hf_xet not available — falling back to default download.")
- # ── Punch-hole support (Linux fallocate) ─────────────────────────────────
- _FALLOC_FL_PUNCH_HOLE = 0x02
- _FALLOC_FL_KEEP_SIZE = 0x01
- _libc = None
- _punch_hole_supported = False
- def _init_punch_hole():
- global _libc, _punch_hole_supported
- if sys.platform != "linux":
- print("[INFO] Punch-hole: not Linux — disabled.")
- return
- try:
- _libc_name = ctypes.util.find_library("c")
- if _libc_name:
- _libc = ctypes.CDLL(_libc_name, use_errno=True)
- tf = os.path.join(WORK_DIR, "_punch_test")
- with open(tf, "wb") as f:
- f.write(b"\xff" * 8192)
- fd = os.open(tf, os.O_RDWR)
- ret = _libc.fallocate(
- ctypes.c_int(fd),
- ctypes.c_int(_FALLOC_FL_PUNCH_HOLE | _FALLOC_FL_KEEP_SIZE),
- ctypes.c_longlong(0),
- ctypes.c_longlong(4096),
- )
- os.close(fd)
- os.remove(tf)
- if ret == 0:
- _punch_hole_supported = True
- print("[OK] Punch-hole (fallocate PUNCH_HOLE): supported.")
- else:
- print(f"[INFO] Punch-hole: fallocate returned {ret} — disabled.")
- else:
- print("[INFO] Punch-hole: libc not found — disabled.")
- except Exception as e:
- print(f"[INFO] Punch-hole: init failed ({e}) — disabled.")
- _init_punch_hole()
- def punch_hole(filepath, offset, length):
- if not _punch_hole_supported or _libc is None or length <= 0:
- return 0
- try:
- fd = os.open(filepath, os.O_RDWR)
- try:
- BS = 4096
- aligned_start = ((offset + BS - 1) // BS) * BS
- aligned_end = ((offset + length) // BS) * BS
- if aligned_end <= aligned_start:
- return 0
- punch_len = aligned_end - aligned_start
- ret = _libc.fallocate(
- ctypes.c_int(fd),
- ctypes.c_int(_FALLOC_FL_PUNCH_HOLE | _FALLOC_FL_KEEP_SIZE),
- ctypes.c_longlong(aligned_start),
- ctypes.c_longlong(punch_len),
- )
- if ret == 0:
- return punch_len
- return 0
- finally:
- os.close(fd)
- except Exception:
- return 0
- def file_blocks_kb(filepath):
- try:
- st = os.stat(filepath)
- return (st.st_blocks * 512) / 1024
- except Exception:
- return -1
- # ── Patch IQ block sizes ─────────────────────────────────────────────────
- _IQ = {
- "IQ1_S":(256,50),"IQ1_M":(256,56),"IQ2_XXS":(256,66),"IQ2_XS":(256,74),
- "IQ2_S":(256,82),"IQ3_XXS":(256,98),"IQ3_S":(256,110),
- "IQ4_NL":(32,18),"IQ4_XS":(256,136),
- }
- for _n,_g in _IQ.items():
- _qt = getattr(GGMLQuantizationType, _n, None)
- if _qt is not None and _qt not in GGML_QUANT_SIZES:
- GGML_QUANT_SIZES[_qt] = _g
- # ── Probe quantize support ──────────────────────────────────────────────
- _QUANTIZE_SUPPORTED = set()
- def _probe_quantize():
- for qt in GGMLQuantizationType:
- if qt in (GGMLQuantizationType.F32, GGMLQuantizationType.F16,
- GGMLQuantizationType.BF16):
- _QUANTIZE_SUPPORTED.add(qt)
- elif qt in GGML_QUANT_SIZES:
- be, _ = GGML_QUANT_SIZES[qt]
- try:
- r = gguf_quants.quantize(np.zeros(be, np.float32), qt)
- if r is not None and len(r.ravel().tobytes()) > 0:
- _QUANTIZE_SUPPORTED.add(qt)
- except Exception: pass
- _probe_quantize()
- print(f"[INFO] quantize() supported: {sorted([q.name for q in _QUANTIZE_SUPPORTED])}")
- # ── Check streaming GGUF write ──────────────────────────────────────────
- _STREAMING = (
- hasattr(GGUFWriter, 'add_tensor_info') and
- hasattr(GGUFWriter, 'write_tensor_data') and
- hasattr(GGUFWriter, 'write_ti_data_to_file')
- )
- print(f"[INFO] GGUF streaming write: {'YES' if _STREAMING else 'NO (buffered)'}")
- # ============================================================================
- # CONFIGURATION
- # ============================================================================
- MODEL_A = {
- "repo": "LuffyTheFox/Qwopus3.5-27B-v3-Uncensored-FernflowerAI-KL-ReLU-GGUF",
- "file": "Qwopus3.5-27B-v3-Uncensored-FernflowerAI-KL-ReLU.Q8_0.gguf",
- }
- MODEL_B = {
- "repo": "LuffyTheFox/FernflowerAI-27B-Uncensored-RYS-KL-ReLU-GGUF",
- "file": "FernflowerAI-27B-Uncensored-RYS-KL-ReLU-Q8_0.gguf",
- }
- MODEL_C = {
- "repo": "LuffyTheFox/Qwen3.5-27B-Uncensored-FernflowerAI-KL-ReLU-GGUF",
- "file": "FernflowerAI-27B-Uncensored-KL-ReLU.Q8_0.gguf",
- }
- MERGE_WEIGHT = 1.0
- OUTPUT_NAME = "Qwopus-3.5-27B-RYS-Uncensored-KL.Q8_0.gguf"
- OUTPUT_PATH = os.path.join(WORK_DIR, OUTPUT_NAME)
- DELTA_PATH = os.path.join(WORK_DIR, "_delta.gguf")
- CHUNK_BYTES = 4 * 1024 * 1024
- # ============================================================================
- # TENSOR CLASSIFICATION
- # ============================================================================
- FINETUNE_SUFFIXES = (
- "attn_q.weight","attn_k.weight","attn_v.weight","attn_output.weight",
- "ssm_in.weight","ssm_out.weight","ssm_a.weight",
- "ssm_d.weight","ssm_dt.weight",
- "ffn_gate.weight","ffn_up.weight","ffn_down.weight",
- "ffn_gate_exps.weight","ffn_up_exps.weight","ffn_down_exps.weight",
- "ffn_gate_inp.weight","ffn_gate_shexp.weight",
- "ffn_up_shexp.weight","ffn_down_shexp.weight",
- "attn_gate.weight",
- )
- PROTECTED_SUBSTRINGS = (
- "token_embd","output.weight","_norm",".bias",
- "rope_freqs","pos_embd",
- "v.blk.","v.patch_embd","v.position_embd",
- "v.post_ln","v.pre_ln","v.output_norm","v.proj",
- "mm.","mtp.",
- )
- FLOAT_QTYPES = {
- GGMLQuantizationType.F32, GGMLQuantizationType.F16, GGMLQuantizationType.BF16,
- }
- # ============================================================================
- # UTILITIES
- # ============================================================================
- def disk_free_gb(p=WORK_DIR):
- st = os.statvfs(p); return (st.f_bavail * st.f_frsize) / 1073741824
- def file_size_gb(p):
- return os.path.getsize(p) / 1073741824 if os.path.exists(p) else 0.0
- def fmt_bytes(n):
- for u in ("B","KB","MB","GB"):
- if n < 1024: return f"{n:.1f} {u}"
- n /= 1024
- return f"{n:.1f} TB"
- def safe_remove(path, protect=None):
- try:
- a = os.path.abspath(path)
- if protect:
- if isinstance(protect, str): protect = [protect]
- for p in protect:
- if a == os.path.abspath(p): return
- if os.path.isfile(a):
- sz = file_size_gb(a)
- os.remove(a)
- print(f"[CLEANUP] Removed {os.path.basename(a)} ({sz:.2f} GB)")
- except OSError: pass
- def flush_and_close(f, label="file"):
- f.flush(); os.fsync(f.fileno()); f.close()
- print(f"[FLUSH] {label}: flush + fsync done.")
- def block_align(qt):
- if qt in FLOAT_QTYPES:
- return {GGMLQuantizationType.F32:4,GGMLQuantizationType.F16:2,
- GGMLQuantizationType.BF16:2}[qt]
- if qt in GGML_QUANT_SIZES: return GGML_QUANT_SIZES[qt][1]
- return 1
- def align_down(v, a):
- return (v // a) * a if a > 0 else v
- def print_workspace():
- print(f"\n[FS] {WORK_DIR}:")
- for fn in sorted(os.listdir(WORK_DIR)):
- fp = os.path.join(WORK_DIR, fn)
- if os.path.isfile(fp):
- logical = file_size_gb(fp)
- actual_kb = file_blocks_kb(fp)
- if actual_kb >= 0:
- actual_gb = actual_kb / (1024*1024)
- saved = logical - actual_gb
- extra = f" (on-disk: {actual_gb:.2f} GB, saved: {saved:.2f} GB)" if saved > 0.01 else ""
- else:
- extra = ""
- print(f" {fn} ({logical:.2f} GB){extra}")
- print(f" Disk free: {disk_free_gb():.1f} GB")
- # ============================================================================
- # CLASSIFICATION
- # ============================================================================
- def is_finetune_weight(name):
- return any(name.endswith(s) for s in FINETUNE_SUFFIXES)
- def is_protected(name):
- return any(p in name for p in PROTECTED_SUBSTRINGS)
- def classify_bc(name, ti_b, ti_c):
- if is_protected(name): return "protect", "structural/norm/embd"
- if not is_finetune_weight(name): return "protect", "not fine-tunable"
- if ti_c is None: return "skip", "missing in C"
- if ti_b["size"] != ti_c["size"]:
- return "skip", f"size mismatch B={ti_b['size']:,} C={ti_c['size']:,}"
- if ti_b["qtype_name"] != ti_c["qtype_name"]:
- return "skip", f"type mismatch B={ti_b['qtype_name']} C={ti_c['qtype_name']}"
- if ti_b["n_elements"] != ti_c["n_elements"]:
- return "skip", f"elem mismatch"
- if ti_b["qtype"] not in _QUANTIZE_SUPPORTED:
- return "skip", f"{ti_b['qtype_name']} no quantize()"
- return "merge", "fine-tunable"
- # ============================================================================
- # GGUF PARSING
- # ============================================================================
- def parse_gguf(path):
- reader = GGUFReader(path)
- tensors = {}
- for t in reader.tensors:
- tensors[t.name] = {
- "offset": int(t.data_offset), "size": int(t.n_bytes),
- "qtype": t.tensor_type, "qtype_name": t.tensor_type.name,
- "n_elements": int(t.n_elements), "shape": list(t.shape),
- }
- n = len(tensors); del reader; gc.collect()
- return tensors, n
- # ============================================================================
- # RAW I/O
- # ============================================================================
- def read_chunk(f, ti, off, sz):
- actual = min(sz, ti["size"] - off)
- if actual <= 0: return b""
- f.seek(ti["offset"] + off); return f.read(actual)
- def write_chunk(f, ti, off, data):
- f.seek(ti["offset"] + off); f.write(data)
- def raw_equal(f1, t1, f2, t2):
- if t1["size"] != t2["size"]: return False
- sz = t1["size"]; off = 0
- while off < sz:
- n = min(CHUNK_BYTES, sz - off)
- f1.seek(t1["offset"]+off); d1 = f1.read(n)
- f2.seek(t2["offset"]+off); d2 = f2.read(n)
- if d1 != d2: return False
- off += n
- return True
- # ============================================================================
- # DEQUANT / QUANT
- # ============================================================================
- def to_f32(raw, qt):
- if qt == GGMLQuantizationType.F32:
- return np.frombuffer(raw, np.float32).copy()
- if qt == GGMLQuantizationType.F16:
- return np.frombuffer(raw, np.float16).astype(np.float32)
- if qt == GGMLQuantizationType.BF16:
- u = np.frombuffer(raw, np.uint16).copy()
- return (u.astype(np.uint32) << 16).view(np.float32).copy()
- return gguf_quants.dequantize(np.frombuffer(raw, np.uint8).copy(), qt
- ).ravel().astype(np.float32)
- def from_f32(f, qt, expected):
- f = np.nan_to_num(f, nan=0., posinf=0., neginf=0.)
- if qt == GGMLQuantizationType.F32:
- r = f.astype(np.float32).tobytes()
- elif qt == GGMLQuantizationType.F16:
- r = f.astype(np.float16).tobytes()
- elif qt == GGMLQuantizationType.BF16:
- r = (f.astype(np.float32).view(np.uint32) >> 16).astype(np.uint16).tobytes()
- elif qt in _QUANTIZE_SUPPORTED:
- be, _ = GGML_QUANT_SIZES[qt]
- n = align_down(len(f), be)
- if n == 0: return None
- try: r = gguf_quants.quantize(f[:n], qt).ravel().tobytes()
- except Exception: return None
- else: return None
- return r if len(r) == expected else None
- def quant_nd(f, qt):
- f = np.nan_to_num(f, nan=0., posinf=0., neginf=0.).astype(np.float32)
- if qt == GGMLQuantizationType.F32: return f.view(np.uint8).copy()
- if qt == GGMLQuantizationType.F16: return f.astype(np.float16).view(np.uint8).copy()
- if qt == GGMLQuantizationType.BF16:
- return (f.view(np.uint32) >> 16).astype(np.uint16).view(np.uint8).copy()
- if qt in _QUANTIZE_SUPPORTED:
- be, _ = GGML_QUANT_SIZES[qt]; n = align_down(len(f), be)
- if n == 0: return np.zeros(0, np.uint8)
- return gguf_quants.quantize(f[:n], qt).ravel().copy()
- return np.zeros(0, np.uint8)
- # ============================================================================
- # DOWNLOAD
- # ============================================================================
- def download(spec, label):
- local_path = os.path.join(WORK_DIR, spec["file"])
- # Skip download if the file already exists in WORK_DIR
- if os.path.isfile(local_path) and os.path.getsize(local_path) > 0:
- print(f"[DL] {label}: already exists — skipping download.")
- print(f"[DL] {label}: {file_size_gb(local_path):.2f} GB -> {local_path}")
- return local_path
- print(f"[DL] {label}: {spec['repo']} / {spec['file']}")
- p = hf_hub_download(spec["repo"], spec["file"], local_dir=WORK_DIR)
- print(f"[DL] {label}: {file_size_gb(p):.2f} GB -> {p}"); return p
- # ============================================================================
- # COMPUTE ONE TENSOR DIFF → quantized uint8 ndarray
- # ============================================================================
- def tensor_diff(f_b, tb, f_c, tc, qt):
- sz = tb["size"]; ba = block_align(qt)
- cb = align_down(CHUNK_BYTES, ba)
- if cb <= 0: cb = ba
- parts = []; off = 0
- while off < sz:
- act = align_down(min(cb, sz - off), ba)
- if act <= 0: break
- rb = read_chunk(f_b, tb, off, act)
- rc = read_chunk(f_c, tc, off, act)
- if len(rb) != act or len(rc) != act: break
- try:
- d = to_f32(rb, qt) - to_f32(rc, qt)
- parts.append(np.nan_to_num(d, nan=0., posinf=0., neginf=0.))
- except Exception:
- if qt in FLOAT_QTYPES:
- eb = {GGMLQuantizationType.F32:4,GGMLQuantizationType.F16:2,
- GGMLQuantizationType.BF16:2}[qt]
- parts.append(np.zeros(act//eb, np.float32))
- elif qt in GGML_QUANT_SIZES:
- be, bs = GGML_QUANT_SIZES[qt]
- parts.append(np.zeros((act//bs)*be, np.float32))
- del rb, rc; off += act
- full = np.concatenate(parts) if parts else np.zeros(tb["n_elements"], np.float32)
- del parts; r = quant_nd(full, qt); del full; return r
- # ============================================================================
- # STAGE 1: COMPUTE DELTA = B - C → GGUF (only changed tensors)
- # + punch holes in B and C after each tensor is processed
- # ============================================================================
- def compute_delta(path_b, path_c, delta_path):
- t0 = time.time()
- print("\n[DELTA] Parsing B and C...")
- ib, nb = parse_gguf(path_b); ic, nc = parse_gguf(path_c)
- print(f"[DELTA] B: {nb} tensors C: {nc} tensors")
- plan = {}; mergeable = []
- for nm in sorted(ib.keys()):
- a, r = classify_bc(nm, ib[nm], ic.get(nm))
- plan[nm] = (a, r)
- if a == "merge": mergeable.append(nm)
- np_ = sum(1 for v in plan.values() if v[0]=="protect")
- ns = sum(1 for v in plan.values() if v[0]=="skip")
- print(f"[DELTA] Mergeable={len(mergeable)} Protected={np_} Skipped={ns}")
- for nm,(a,r) in sorted(plan.items()):
- if a=="skip": print(f" SKIP: {nm:<48} {r}")
- # ── Punch holes for protected/skipped tensors immediately ────────────
- if _punch_hole_supported:
- total_punched = 0
- for nm in sorted(ib.keys()):
- act, _ = plan.get(nm, ("protect",""))
- if act != "merge":
- tb = ib[nm]
- p = punch_hole(path_b, tb["offset"], tb["size"])
- total_punched += p
- tc = ic.get(nm)
- if tc is not None:
- p = punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
- total_punched += p
- if total_punched > 0:
- print(f"[PUNCH] Freed {fmt_bytes(total_punched)} from "
- f"protected/skipped tensors in B and C")
- print(f"[PUNCH] Disk free: {disk_free_gb():.1f} GB")
- if not mergeable:
- w = GGUFWriter(delta_path, arch="delta")
- w.add_description("Empty delta")
- w.write_header_to_file(); w.write_kv_data_to_file()
- w.write_ti_data_to_file(); w.close()
- print("[DELTA] Empty delta created."); return {}
- # ── Pass 1: raw byte compare ─────────────────────────────────────────
- print(f"\n[DELTA] Pass 1: raw byte compare ({len(mergeable)} tensors)...")
- fb = open(path_b,"rb"); fc = open(path_c,"rb")
- changed=[]; same=[]
- for i,nm in enumerate(mergeable):
- eq = raw_equal(fb, ib[nm], fc, ic[nm])
- if eq:
- same.append(nm)
- if _punch_hole_supported:
- punch_hole(path_b, ib[nm]["offset"], ib[nm]["size"])
- punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
- else:
- changed.append(nm)
- if (i+1)%100==0 or i+1==len(mergeable):
- print(f" {i+1}/{len(mergeable)} changed={len(changed)} same={len(same)}"
- f" disk_free={disk_free_gb():.1f}GB")
- fb.close(); fc.close()
- print(f"[DELTA] Result: {len(changed)} changed, {len(same)} identical")
- if _punch_hole_supported and same:
- print(f"[PUNCH] Punched {len(same)} identical tensors from B and C")
- print(f"[PUNCH] Disk free: {disk_free_gb():.1f} GB")
- if not changed:
- w = GGUFWriter(delta_path, arch="delta")
- w.add_description("Empty delta: B==C")
- w.write_header_to_file(); w.write_kv_data_to_file()
- w.write_ti_data_to_file(); w.close()
- print("[DELTA] Empty delta."); return {"changed":0,"unchanged":len(same)}
- # ── Write delta GGUF ─────────────────────────────────────────────────
- w = GGUFWriter(delta_path, arch="delta")
- w.add_description(f"Delta (B-C): {len(changed)} changed tensors. "
- f"B={MODEL_B['repo']}/{MODEL_B['file']} "
- f"C={MODEL_C['repo']}/{MODEL_C['file']}")
- w.add_uint32("delta.changed_count", len(changed))
- w.add_uint32("delta.unchanged_count", len(same))
- fb = open(path_b,"rb"); fc = open(path_c,"rb")
- if _STREAMING:
- for nm in changed:
- ti = ib[nm]
- w.add_tensor_info(name=nm, tensor_shape=ti["shape"],
- tensor_dtype=np.float32,
- tensor_nbytes=ti["size"], raw_dtype=ti["qtype"])
- w.write_header_to_file()
- w.write_kv_data_to_file()
- w.write_ti_data_to_file()
- print(f"\n[DELTA] Pass 2: compute + stream {len(changed)} deltas...")
- print(f"{'#':>4} {'Tensor':<52} {'Type':<10} {'Size':>9} {'Punched':>10}")
- print("-"*95)
- total_punched_p2 = 0
- for i,nm in enumerate(changed):
- d = tensor_diff(fb, ib[nm], fc, ic[nm], ib[nm]["qtype"])
- w.write_tensor_data(d)
- pb = pc = 0
- if _punch_hole_supported:
- pb = punch_hole(path_b, ib[nm]["offset"], ib[nm]["size"])
- pc = punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
- total_punched_p2 += pb + pc
- print(f"{i+1:>4} {nm:<52} {ib[nm]['qtype_name']:<10} "
- f"{fmt_bytes(ib[nm]['size']):>9} {fmt_bytes(pb+pc):>10}")
- del d
- if (i+1)%30==0: gc.collect()
- if total_punched_p2 > 0:
- print(f"\n[PUNCH] Pass 2 freed: {fmt_bytes(total_punched_p2)}")
- print(f"[PUNCH] Disk free: {disk_free_gb():.1f} GB")
- else:
- print(f"\n[DELTA] Pass 2: compute + buffer {len(changed)} deltas...")
- print("[DELTA] WARNING: buffered mode — high RAM usage!")
- print(f"{'#':>4} {'Tensor':<52} {'Type':<10} {'Size':>9} {'Punched':>10}")
- print("-"*95)
- total_punched_p2 = 0
- for i,nm in enumerate(changed):
- ti = ib[nm]
- d = tensor_diff(fb, ti, fc, ic[nm], ti["qtype"])
- w.add_tensor(nm, d, raw_shape=np.array(ti["shape"],np.uint64),
- raw_dtype=ti["qtype"])
- pb = pc = 0
- if _punch_hole_supported:
- pb = punch_hole(path_b, ib[nm]["offset"], ib[nm]["size"])
- pc = punch_hole(path_c, ic[nm]["offset"], ic[nm]["size"])
- total_punched_p2 += pb + pc
- print(f"{i+1:>4} {nm:<52} {ti['qtype_name']:<10} "
- f"{fmt_bytes(ti['size']):>9} {fmt_bytes(pb+pc):>10}")
- del d
- if (i+1)%30==0: gc.collect()
- w.write_header_to_file()
- w.write_kv_data_to_file()
- w.write_tensors_to_file()
- if total_punched_p2 > 0:
- print(f"\n[PUNCH] Pass 2 freed: {fmt_bytes(total_punched_p2)}")
- fb.close(); fc.close(); w.close()
- el = time.time()-t0
- print(f"\n[DELTA] Done: {len(changed)} tensors in delta GGUF "
- f"({file_size_gb(delta_path):.2f} GB, {el:.1f}s)")
- print(f"[DELTA] Disk free: {disk_free_gb():.1f} GB")
- return {"changed":len(changed),"unchanged":len(same)}
- # ============================================================================
- # STAGE 2: APPLY DELTA TO A → output = A + delta * weight
- # A is RENAMED to output (no copy), then modified in-place
- # ============================================================================
- def apply_delta(path_a, delta_path, weight, output_path):
- t0 = time.time()
- print("\n[APPLY] Parsing delta GGUF...")
- id_, nd = parse_gguf(delta_path)
- print(f"[APPLY] Delta: {nd} changed tensors")
- if nd == 0:
- print("[APPLY] Empty delta → renaming A as output.")
- shutil.move(path_a, output_path); return {}
- print("[APPLY] Parsing A...")
- ia, na = parse_gguf(path_a)
- sa = os.path.getsize(path_a)
- print(f"[APPLY] A: {na} tensors ({sa:,} B)")
- miss = [n for n in id_ if n not in ia]
- if miss:
- print(f"[APPLY] WARNING: {len(miss)} delta tensors missing in A")
- for n in miss[:5]: print(f" {n}")
- # ── In-place: rename A → output (no disk space needed) ───────────────
- abs_a = os.path.abspath(path_a)
- abs_o = os.path.abspath(output_path)
- if abs_a == abs_o:
- print(f"[APPLY] A and output are the same file — modifying in-place.")
- else:
- print(f"[APPLY] Renaming A → {os.path.basename(output_path)} (no copy, 0 bytes needed)...")
- shutil.move(path_a, output_path)
- assert os.path.exists(output_path), "Rename failed"
- print(f"[APPLY] Rename OK — disk free: {disk_free_gb():.1f} GB")
- fo = open(output_path, "r+b")
- fd = open(delta_path, "rb")
- # Architecture info
- attn=set(); ssm=set(); ffn=set()
- for nm in ia:
- p = nm.split(".")
- if len(p)>=3 and p[0]=="blk":
- b = f"blk.{p[1]}"
- if ".attn_q." in nm: attn.add(b)
- if ".ssm_in." in nm: ssm.add(b)
- if ".ffn_gate." in nm: ffn.add(b)
- print(f"\n[APPLY] Arch: attn={len(attn)} deltanet={len(ssm)} ffn={len(ffn)}")
- names = sorted(ia.keys())
- print(f"\n[APPLY] output = A + Δ * {weight} ({nd} deltas in {na} tensors)")
- print(f"{'#':>4} {'Tensor':<52} {'Type':<10} {'Size':>9} {'Action':<10} Detail")
- print("-"*125)
- st = {"merged":0,"protected":0,"skipped":0,"total_changed":0,
- "identical":0,"failed":0,"attn":0,"ssm":0,"ffn":0}
- for idx, nm in enumerate(names):
- ta = ia[nm]; qn = ta["qtype_name"]; sz = ta["size"]
- if nm not in id_:
- if is_protected(nm) or not is_finetune_weight(nm):
- st["protected"]+=1; lbl="PROTECT"
- rsn = "structural/norm/embd" if is_protected(nm) else "not fine-tunable"
- else:
- st["skipped"]+=1; lbl="KEEP"; rsn="unchanged (not in delta)"
- print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {lbl:<10} {rsn}")
- continue
- td = id_[nm]
- if ta["qtype"] != td["qtype"]:
- st["skipped"]+=1
- print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'SKIP':<10} "
- f"qtype mismatch A={ta['qtype_name']} Δ={td['qtype_name']}"); continue
- if ta["size"] != td["size"]:
- st["skipped"]+=1
- print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'SKIP':<10} "
- f"size mismatch A={ta['size']} Δ={td['size']}"); continue
- qt = ta["qtype"]; ds = ta["size"]
- ba = block_align(qt); cb = align_down(CHUNK_BYTES, ba)
- if cb<=0: cb=ba
- chg=0; fail=0; off=0
- while off < ds:
- act = align_down(min(cb, ds-off), ba)
- if act<=0: break
- ra = read_chunk(fo, ta, off, act)
- rd = read_chunk(fd, td, off, act)
- if len(ra)!=act or len(rd)!=act: break
- try:
- fa = to_f32(ra, qt); fdd = to_f32(rd, qt)
- except Exception:
- fail+=1; del ra,rd; off+=act; continue
- n = min(len(fa), len(fdd))
- m = fa[:n] + fdd[:n] * np.float32(weight)
- del fa, fdd
- nr = from_f32(m, qt, len(ra)); del m
- if nr is not None:
- oa = np.frombuffer(ra, np.uint8)
- na_ = np.frombuffer(nr, np.uint8)
- chg += int(np.sum(oa != na_))
- write_chunk(fo, ta, off, nr)
- del oa, na_, nr
- else: fail+=1
- del ra, rd; off += act
- st["merged"]+=1; st["total_changed"]+=chg; st["failed"]+=fail
- if ".attn_" in nm and "_norm" not in nm: st["attn"]+=1
- elif ".ssm_" in nm and "_norm" not in nm: st["ssm"]+=1
- elif ".ffn_" in nm and "_norm" not in nm: st["ffn"]+=1
- if chg>0:
- pct=chg/sz*100; fs=f" ({fail} kept)" if fail else ""
- print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'MERGED':<10} "
- f"{chg:,}B ({pct:.2f}%){fs}")
- else:
- st["identical"]+=1
- print(f"{idx+1:>4} {nm:<52} {qn:<10} {fmt_bytes(sz):>9} {'MERGED':<10} "
- f"(identical)")
- if (idx+1)%50==0: gc.collect()
- flush_and_close(fo, "output"); fd.close()
- el = time.time()-t0
- print(f"\n{'='*75}\nAPPLY SUMMARY\n{'='*75}")
- print(f"Protected: {st['protected']}")
- print(f"Merged: {st['merged']} (attn={st['attn']} ssm={st['ssm']} ffn={st['ffn']})")
- print(f" Identical: {st['identical']}")
- print(f" Changed: {st['total_changed']:,} bytes")
- if st['failed']: print(f" Failed chk: {st['failed']}")
- print(f"Kept/Skipped: {st['skipped']}")
- print(f"Time: {el:.1f}s")
- print(f"Output: {os.path.basename(output_path)} ({file_size_gb(output_path):.2f} GB)")
- return st
- # ============================================================================
- # VERIFY
- # ============================================================================
- def verify(path):
- print("\n[VERIFY] Checking output...")
- try: info, n = parse_gguf(path); print(f"[VERIFY] Parsed {n} tensors.")
- except Exception as e: print(f"[VERIFY] ERROR: {e}"); return False
- f = open(path,"rb"); ok=w=0
- for nm in sorted(info.keys())[:30]:
- ti = info[nm]; cs = min(4096, ti["size"])
- raw = read_chunk(f, ti, 0, cs)
- a = np.frombuffer(raw, np.uint8)
- if a.sum()==0 and cs>64: w+=1; print(f" WARNING: '{nm}' all zeros!")
- else: ok+=1
- del raw, a
- f.close()
- if w: print(f"[VERIFY] {w} zero tensors.")
- print(f"[VERIFY] {ok} tensors OK."); return w==0
- # ============================================================================
- # MAIN
- # ============================================================================
- def main():
- print("="*75)
- print("GGUF ADD-DIFFERENCE MERGE — Disk-Optimized")
- print("output = A + (B - C) * weight")
- print()
- print("Pipeline:")
- print(" 1) Download B and C")
- print(" 2) Compute delta = B - C → GGUF (punch holes in B/C)")
- print(" 3) Delete B and C")
- print(" 4) Download A")
- print(" 5) Rename A → output, apply delta in-place (no copy)")
- print(" 6) Delete delta")
- print()
- print(f"WORK_DIR: {WORK_DIR}")
- print(f"OUTPUT: {OUTPUT_PATH}")
- print(f"DELTA: {DELTA_PATH}")
- print(f"WEIGHT: {MERGE_WEIGHT}")
- print(f"PUNCH_HOLE: {'YES' if _punch_hole_supported else 'NO'}")
- print("="*75)
- # 1) Download B and C
- print("\n"+"="*75+"\nSTEP 1: Download B and C\n"+"="*75)
- pb = download(MODEL_B, "B (fine-tuned)")
- pc = download(MODEL_C, "C (reference)")
- print_workspace()
- # 2) Compute delta
- print("\n"+"="*75+"\nSTEP 2: Compute delta = B - C → GGUF\n"+"="*75)
- compute_delta(pb, pc, DELTA_PATH)
- gc.collect(); print_workspace()
- # 3) Delete B and C
- print("\n"+"="*75+"\nSTEP 3: Delete B and C\n"+"="*75)
- for p in set([os.path.abspath(pb), os.path.abspath(pc)]):
- safe_remove(p, protect=[OUTPUT_PATH, DELTA_PATH])
- gc.collect(); print_workspace()
- # 4) Download A
- print("\n"+"="*75+"\nSTEP 4: Download A\n"+"="*75)
- pa = download(MODEL_A, "A (base)")
- print_workspace()
- # 5) Apply delta (in-place, no copy)
- print("\n"+"="*75+"\nSTEP 5: Apply output = A + delta * weight (in-place)\n"+"="*75)
- apply_delta(pa, DELTA_PATH, MERGE_WEIGHT, OUTPUT_PATH)
- gc.collect()
- # 6) Cleanup
- print("\n"+"="*75+"\nSTEP 6: Cleanup\n"+"="*75)
- # A was already renamed to output — only delete delta
- safe_remove(DELTA_PATH, protect=OUTPUT_PATH)
- gc.collect(); print_workspace()
- # Verify
- print("\n"+"="*75+"\nVERIFY\n"+"="*75)
- verify(OUTPUT_PATH)
- # Done
- print("\n"+"="*75+"\nDONE\n"+"="*75)
- print_workspace()
- print(f"\n Final: {OUTPUT_PATH}")
- print(f" Size: {file_size_gb(OUTPUT_PATH):.2f} GB")
- print(f" Disk free: {disk_free_gb():.1f} GB")
- main()
Advertisement
Add Comment
Please, Sign In to add comment