Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # ============================================================
- # gguf_q8_to_fp8_safetensors.py
- # Converts Qwen3-30B-A3B (qwen3moe) Q8_0 GGUF → fp8_e4m3fn safetensors
- # Google Colab Free Tier (12 GB RAM) — streaming, no full model in RAM
- # ============================================================
- # !pip install gguf safetensors torch numpy -q
- # ============================================================
- import os
- import gc
- import json
- import struct
- import glob
- import numpy as np
- import torch
- import psutil
- from typing import List, Dict, Tuple
- from gguf import GGUFReader, dequantize
- # ------------------------------------------------------------------
- # Paths — auto-detects the first .gguf in workspace
- # ------------------------------------------------------------------
- WORKSPACE = "/content/gguf_workspace"
- _gguf_matches = glob.glob(os.path.join(WORKSPACE, "*.gguf"))
- if not _gguf_matches:
- raise FileNotFoundError(f"No .gguf file found in {WORKSPACE}")
- GGUF_PATH = _gguf_matches[0]
- OUTPUT_PATH = os.path.join(
- WORKSPACE,
- os.path.splitext(os.path.basename(GGUF_PATH))[0] + "-fp8_e4m3fn.safetensors"
- )
- print(f"Input : {GGUF_PATH}")
- print(f"Output : {OUTPUT_PATH}")
- # ------------------------------------------------------------------
- # FP8 constants
- # ------------------------------------------------------------------
- FP8_E4M3FN_MAX = 448.0
- FP8_DTYPE = torch.float8_e4m3fn
- FP8_DTYPE_STR = "F8_E4M3"
- F32_DTYPE_STR = "F32"
- # These layers stay float32 — too sensitive or out of fp8 range
- FP32_KEEP_SUFFIXES = (
- "norm.weight",
- "norm_weight",
- "embed_tokens.weight",
- "lm_head.weight",
- "bias",
- )
- # Tensors with more elements than this get chunked during fp8 quantization
- # to cap peak RAM. 64M float32 elements = 256 MB per chunk.
- CHUNK_ELEMS = 64 * 1024 * 1024
- # ------------------------------------------------------------------
- # Memory monitor
- # ------------------------------------------------------------------
- def _ram_gb() -> float:
- return psutil.Process(os.getpid()).memory_info().rss / 1e9
- # ------------------------------------------------------------------
- # Helpers
- # ------------------------------------------------------------------
- def _dtype_nbytes(dtype_str: str, n_elements: int) -> int:
- return n_elements * {"F8_E4M3": 1, "F32": 4, "F16": 2, "BF16": 2}[dtype_str]
- def _should_keep_fp32(name: str) -> bool:
- return any(name.endswith(s) for s in FP32_KEEP_SUFFIXES)
- def _n_elements(shape: List[int]) -> int:
- n = 1
- for s in shape:
- n *= s
- return n
- def _quantize_fp8_inplace(arr_f32: np.ndarray) -> Tuple[bytes, np.ndarray]:
- """
- Quantize a float32 numpy array to FP8-E4M3FN.
- For large tensors, processes in row-chunks to limit peak RAM.
- Returns (fp8_raw_bytes, scale_f32_array).
- Scale is per-tensor (computed over full array first, cheaply).
- """
- # Compute global amax with numpy — no torch copy needed
- amax = float(np.max(np.abs(arr_f32)))
- scale = max(amax / FP8_E4M3FN_MAX, 1e-12)
- scale_arr = np.array([scale], dtype=np.float32)
- n_elems = arr_f32.size
- if n_elems <= CHUNK_ELEMS:
- # Small tensor — one shot
- t = torch.from_numpy(arr_f32) # zero-copy
- fp8 = (t / scale).clamp(-FP8_E4M3FN_MAX, FP8_E4M3FN_MAX).to(FP8_DTYPE)
- raw = fp8.view(torch.uint8).numpy().tobytes()
- del t, fp8
- return raw, scale_arr
- # Large tensor — chunk along first axis to cap RAM
- flat = arr_f32.reshape(-1) # view, no copy
- chunk_size = CHUNK_ELEMS
- parts = []
- for start in range(0, n_elems, chunk_size):
- chunk = flat[start : start + chunk_size] # view
- t = torch.from_numpy(chunk) # zero-copy
- fp8 = (t / scale).clamp(-FP8_E4M3FN_MAX, FP8_E4M3FN_MAX).to(FP8_DTYPE)
- parts.append(fp8.view(torch.uint8).numpy().tobytes())
- del t, fp8
- return b"".join(parts), scale_arr
- # ------------------------------------------------------------------
- # Pass 1 — build tensor plan (metadata only, no dequantisation)
- # ------------------------------------------------------------------
- def build_plan(reader: GGUFReader) -> List[Dict]:
- plan = []
- for tensor in reader.tensors:
- name = tensor.name
- shape = tensor.shape.tolist()
- ne = _n_elements(shape)
- if _should_keep_fp32(name):
- plan.append({
- "name": name,
- "shape": shape,
- "dtype_str": F32_DTYPE_STR,
- "nbytes": _dtype_nbytes(F32_DTYPE_STR, ne),
- "is_scale": False,
- "keep_f32": True,
- })
- else:
- plan.append({
- "name": name,
- "shape": shape,
- "dtype_str": FP8_DTYPE_STR,
- "nbytes": _dtype_nbytes(FP8_DTYPE_STR, ne),
- "is_scale": False,
- "keep_f32": False,
- })
- plan.append({
- "name": f"{name}_scale",
- "shape": [1],
- "dtype_str": F32_DTYPE_STR,
- "nbytes": 4,
- "is_scale": True,
- "keep_f32": False,
- })
- return plan
- # ------------------------------------------------------------------
- # Build safetensors header
- # ------------------------------------------------------------------
- def build_header(plan: List[Dict], metadata: Dict[str, str]) -> bytes:
- header: Dict = {"__metadata__": metadata}
- offset = 0
- for entry in plan:
- nb = entry["nbytes"]
- header[entry["name"]] = {
- "dtype": entry["dtype_str"],
- "shape": entry["shape"],
- "data_offsets": [offset, offset + nb],
- }
- offset += nb
- raw = json.dumps(header, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
- pad = (8 - len(raw) % 8) % 8
- return raw + b" " * pad
- # ------------------------------------------------------------------
- # Main conversion — streaming two-pass
- # ------------------------------------------------------------------
- def convert(gguf_path: str, output_path: str) -> None:
- print(f"\n[1/3] Opening GGUF... RAM: {_ram_gb():.2f} GB")
- reader = GGUFReader(gguf_path)
- # Pass 1
- plan = build_plan(reader)
- total_data = sum(e["nbytes"] for e in plan)
- print(f"[1/3] {len(reader.tensors)} raw tensors → {len(plan)} safetensors entries")
- print(f" Estimated output: {total_data / 1e9:.2f} GB")
- # Header
- arch_field = reader.get_field("general.architecture")
- arch_str = str(arch_field.parts[-1].tolist()[0]) if arch_field else "qwen3moe"
- metadata = {
- "architecture": arch_str,
- "format": "fp8_e4m3fn_scaled",
- "source": "converted from Q8_0 GGUF",
- "scale_suffix": "_scale",
- }
- header_bytes = build_header(plan, metadata)
- print(f"[2/3] Header built: {len(header_bytes)} bytes")
- # Pass 2 — stream write
- name_to_idx = {t.name: i for i, t in enumerate(reader.tensors)}
- scale_to_base = {
- e["name"]: e["name"][: -len("_scale")]
- for e in plan if e["is_scale"]
- }
- _scale_cache: Dict[str, np.ndarray] = {}
- print(f"[3/3] Writing output... RAM: {_ram_gb():.2f} GB")
- with open(output_path, "wb") as f:
- f.write(struct.pack("<Q", len(header_bytes)))
- f.write(header_bytes)
- for i, entry in enumerate(plan):
- name = entry["name"]
- # ── Scale entry ───────────────────────────────────────
- if entry["is_scale"]:
- base = scale_to_base[name]
- scale_a = _scale_cache.pop(base)
- f.write(scale_a.tobytes())
- del scale_a
- continue
- # ── Dequantize from Q8_0 ──────────────────────────────
- raw = reader.tensors[name_to_idx[name]]
- weights = dequantize(raw.data, raw.tensor_type)
- # astype makes a new contiguous float32 array;
- # delete the dequantized array immediately after
- arr_f32 = weights.astype(np.float32)
- del weights
- gc.collect()
- # ── Write ─────────────────────────────────────────────
- if entry["keep_f32"]:
- f.write(arr_f32.tobytes()) # direct — no torch needed
- del arr_f32
- else:
- # Quantize → get bytes + scale; arr_f32 freed inside chunked fn
- fp8_bytes, scale_arr = _quantize_fp8_inplace(arr_f32)
- del arr_f32 # free float32 BEFORE write
- gc.collect()
- f.write(fp8_bytes)
- del fp8_bytes
- _scale_cache[name] = scale_arr
- gc.collect()
- if (i + 1) % 50 == 0 or (i + 1) == len(plan):
- pct = (i + 1) / len(plan) * 100
- print(
- f" {i+1}/{len(plan)} ({pct:.1f}%) RAM: {_ram_gb():.2f} GB",
- flush=True,
- )
- size_gb = os.path.getsize(output_path) / 1e9
- print(f"\nDone. Output: {output_path} ({size_gb:.3f} GB)")
- # ------------------------------------------------------------------
- # Run
- # ------------------------------------------------------------------
- convert(GGUF_PATH, OUTPUT_PATH)
Advertisement