LuffyTheFox

Untitled

Apr 25th, 2026
128
0
Never
9
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 9.81 KB | None | 0 0
  1. # ============================================================
  2. # gguf_q8_to_fp8_safetensors.py
  3. # Converts Qwen3-30B-A3B (qwen3moe) Q8_0 GGUF → fp8_e4m3fn safetensors
  4. # Google Colab Free Tier (12 GB RAM) — streaming, no full model in RAM
  5. # ============================================================
  6. # !pip install gguf safetensors torch numpy -q
  7. # ============================================================
  8.  
  9. import os
  10. import gc
  11. import json
  12. import struct
  13. import glob
  14. import numpy as np
  15. import torch
  16. import psutil
  17. from typing import List, Dict, Tuple
  18. from gguf import GGUFReader, dequantize
  19.  
  20. # ------------------------------------------------------------------
  21. # Paths — auto-detects the first .gguf in workspace
  22. # ------------------------------------------------------------------
  23. WORKSPACE = "/content/gguf_workspace"
  24. _gguf_matches = glob.glob(os.path.join(WORKSPACE, "*.gguf"))
  25. if not _gguf_matches:
  26. raise FileNotFoundError(f"No .gguf file found in {WORKSPACE}")
  27. GGUF_PATH = _gguf_matches[0]
  28. OUTPUT_PATH = os.path.join(
  29. WORKSPACE,
  30. os.path.splitext(os.path.basename(GGUF_PATH))[0] + "-fp8_e4m3fn.safetensors"
  31. )
  32.  
  33. print(f"Input : {GGUF_PATH}")
  34. print(f"Output : {OUTPUT_PATH}")
  35.  
  36. # ------------------------------------------------------------------
  37. # FP8 constants
  38. # ------------------------------------------------------------------
  39. FP8_E4M3FN_MAX = 448.0
  40. FP8_DTYPE = torch.float8_e4m3fn
  41. FP8_DTYPE_STR = "F8_E4M3"
  42. F32_DTYPE_STR = "F32"
  43.  
  44. # These layers stay float32 — too sensitive or out of fp8 range
  45. FP32_KEEP_SUFFIXES = (
  46. "norm.weight",
  47. "norm_weight",
  48. "embed_tokens.weight",
  49. "lm_head.weight",
  50. "bias",
  51. )
  52.  
  53. # Tensors with more elements than this get chunked during fp8 quantization
  54. # to cap peak RAM. 64M float32 elements = 256 MB per chunk.
  55. CHUNK_ELEMS = 64 * 1024 * 1024
  56.  
  57. # ------------------------------------------------------------------
  58. # Memory monitor
  59. # ------------------------------------------------------------------
  60.  
  61. def _ram_gb() -> float:
  62. return psutil.Process(os.getpid()).memory_info().rss / 1e9
  63.  
  64. # ------------------------------------------------------------------
  65. # Helpers
  66. # ------------------------------------------------------------------
  67.  
  68. def _dtype_nbytes(dtype_str: str, n_elements: int) -> int:
  69. return n_elements * {"F8_E4M3": 1, "F32": 4, "F16": 2, "BF16": 2}[dtype_str]
  70.  
  71.  
  72. def _should_keep_fp32(name: str) -> bool:
  73. return any(name.endswith(s) for s in FP32_KEEP_SUFFIXES)
  74.  
  75.  
  76. def _n_elements(shape: List[int]) -> int:
  77. n = 1
  78. for s in shape:
  79. n *= s
  80. return n
  81.  
  82.  
  83. def _quantize_fp8_inplace(arr_f32: np.ndarray) -> Tuple[bytes, np.ndarray]:
  84. """
  85. Quantize a float32 numpy array to FP8-E4M3FN.
  86. For large tensors, processes in row-chunks to limit peak RAM.
  87. Returns (fp8_raw_bytes, scale_f32_array).
  88. Scale is per-tensor (computed over full array first, cheaply).
  89. """
  90. # Compute global amax with numpy — no torch copy needed
  91. amax = float(np.max(np.abs(arr_f32)))
  92. scale = max(amax / FP8_E4M3FN_MAX, 1e-12)
  93. scale_arr = np.array([scale], dtype=np.float32)
  94.  
  95. n_elems = arr_f32.size
  96.  
  97. if n_elems <= CHUNK_ELEMS:
  98. # Small tensor — one shot
  99. t = torch.from_numpy(arr_f32) # zero-copy
  100. fp8 = (t / scale).clamp(-FP8_E4M3FN_MAX, FP8_E4M3FN_MAX).to(FP8_DTYPE)
  101. raw = fp8.view(torch.uint8).numpy().tobytes()
  102. del t, fp8
  103. return raw, scale_arr
  104.  
  105. # Large tensor — chunk along first axis to cap RAM
  106. flat = arr_f32.reshape(-1) # view, no copy
  107. chunk_size = CHUNK_ELEMS
  108. parts = []
  109. for start in range(0, n_elems, chunk_size):
  110. chunk = flat[start : start + chunk_size] # view
  111. t = torch.from_numpy(chunk) # zero-copy
  112. fp8 = (t / scale).clamp(-FP8_E4M3FN_MAX, FP8_E4M3FN_MAX).to(FP8_DTYPE)
  113. parts.append(fp8.view(torch.uint8).numpy().tobytes())
  114. del t, fp8
  115.  
  116. return b"".join(parts), scale_arr
  117.  
  118.  
  119. # ------------------------------------------------------------------
  120. # Pass 1 — build tensor plan (metadata only, no dequantisation)
  121. # ------------------------------------------------------------------
  122.  
  123. def build_plan(reader: GGUFReader) -> List[Dict]:
  124. plan = []
  125. for tensor in reader.tensors:
  126. name = tensor.name
  127. shape = tensor.shape.tolist()
  128. ne = _n_elements(shape)
  129.  
  130. if _should_keep_fp32(name):
  131. plan.append({
  132. "name": name,
  133. "shape": shape,
  134. "dtype_str": F32_DTYPE_STR,
  135. "nbytes": _dtype_nbytes(F32_DTYPE_STR, ne),
  136. "is_scale": False,
  137. "keep_f32": True,
  138. })
  139. else:
  140. plan.append({
  141. "name": name,
  142. "shape": shape,
  143. "dtype_str": FP8_DTYPE_STR,
  144. "nbytes": _dtype_nbytes(FP8_DTYPE_STR, ne),
  145. "is_scale": False,
  146. "keep_f32": False,
  147. })
  148. plan.append({
  149. "name": f"{name}_scale",
  150. "shape": [1],
  151. "dtype_str": F32_DTYPE_STR,
  152. "nbytes": 4,
  153. "is_scale": True,
  154. "keep_f32": False,
  155. })
  156. return plan
  157.  
  158.  
  159. # ------------------------------------------------------------------
  160. # Build safetensors header
  161. # ------------------------------------------------------------------
  162.  
  163. def build_header(plan: List[Dict], metadata: Dict[str, str]) -> bytes:
  164. header: Dict = {"__metadata__": metadata}
  165. offset = 0
  166. for entry in plan:
  167. nb = entry["nbytes"]
  168. header[entry["name"]] = {
  169. "dtype": entry["dtype_str"],
  170. "shape": entry["shape"],
  171. "data_offsets": [offset, offset + nb],
  172. }
  173. offset += nb
  174. raw = json.dumps(header, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
  175. pad = (8 - len(raw) % 8) % 8
  176. return raw + b" " * pad
  177.  
  178.  
  179. # ------------------------------------------------------------------
  180. # Main conversion — streaming two-pass
  181. # ------------------------------------------------------------------
  182.  
  183. def convert(gguf_path: str, output_path: str) -> None:
  184. print(f"\n[1/3] Opening GGUF... RAM: {_ram_gb():.2f} GB")
  185. reader = GGUFReader(gguf_path)
  186.  
  187. # Pass 1
  188. plan = build_plan(reader)
  189. total_data = sum(e["nbytes"] for e in plan)
  190. print(f"[1/3] {len(reader.tensors)} raw tensors → {len(plan)} safetensors entries")
  191. print(f" Estimated output: {total_data / 1e9:.2f} GB")
  192.  
  193. # Header
  194. arch_field = reader.get_field("general.architecture")
  195. arch_str = str(arch_field.parts[-1].tolist()[0]) if arch_field else "qwen3moe"
  196. metadata = {
  197. "architecture": arch_str,
  198. "format": "fp8_e4m3fn_scaled",
  199. "source": "converted from Q8_0 GGUF",
  200. "scale_suffix": "_scale",
  201. }
  202. header_bytes = build_header(plan, metadata)
  203. print(f"[2/3] Header built: {len(header_bytes)} bytes")
  204.  
  205. # Pass 2 — stream write
  206. name_to_idx = {t.name: i for i, t in enumerate(reader.tensors)}
  207. scale_to_base = {
  208. e["name"]: e["name"][: -len("_scale")]
  209. for e in plan if e["is_scale"]
  210. }
  211. _scale_cache: Dict[str, np.ndarray] = {}
  212.  
  213. print(f"[3/3] Writing output... RAM: {_ram_gb():.2f} GB")
  214. with open(output_path, "wb") as f:
  215. f.write(struct.pack("<Q", len(header_bytes)))
  216. f.write(header_bytes)
  217.  
  218. for i, entry in enumerate(plan):
  219. name = entry["name"]
  220.  
  221. # ── Scale entry ───────────────────────────────────────
  222. if entry["is_scale"]:
  223. base = scale_to_base[name]
  224. scale_a = _scale_cache.pop(base)
  225. f.write(scale_a.tobytes())
  226. del scale_a
  227. continue
  228.  
  229. # ── Dequantize from Q8_0 ──────────────────────────────
  230. raw = reader.tensors[name_to_idx[name]]
  231. weights = dequantize(raw.data, raw.tensor_type)
  232. # astype makes a new contiguous float32 array;
  233. # delete the dequantized array immediately after
  234. arr_f32 = weights.astype(np.float32)
  235. del weights
  236. gc.collect()
  237.  
  238. # ── Write ─────────────────────────────────────────────
  239. if entry["keep_f32"]:
  240. f.write(arr_f32.tobytes()) # direct — no torch needed
  241. del arr_f32
  242.  
  243. else:
  244. # Quantize → get bytes + scale; arr_f32 freed inside chunked fn
  245. fp8_bytes, scale_arr = _quantize_fp8_inplace(arr_f32)
  246. del arr_f32 # free float32 BEFORE write
  247. gc.collect()
  248. f.write(fp8_bytes)
  249. del fp8_bytes
  250. _scale_cache[name] = scale_arr
  251.  
  252. gc.collect()
  253.  
  254. if (i + 1) % 50 == 0 or (i + 1) == len(plan):
  255. pct = (i + 1) / len(plan) * 100
  256. print(
  257. f" {i+1}/{len(plan)} ({pct:.1f}%) RAM: {_ram_gb():.2f} GB",
  258. flush=True,
  259. )
  260.  
  261. size_gb = os.path.getsize(output_path) / 1e9
  262. print(f"\nDone. Output: {output_path} ({size_gb:.3f} GB)")
  263.  
  264.  
  265. # ------------------------------------------------------------------
  266. # Run
  267. # ------------------------------------------------------------------
  268.  
  269. convert(GGUF_PATH, OUTPUT_PATH)
Advertisement
Comments
  • User was banned
  • User was banned
  • User was banned
  • User was banned
  • User was banned
  • User was banned
  • User was banned
  • User was banned
  • User was banned
Add Comment
Please, Sign In to add comment