LuffyTheFox

Untitled

Mar 26th, 2026 (edited)
377
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 10.74 KB | None | 0 0
  1. # ============================================================================
  2. # GGUF Quantize: Download Q8_0 → Quantize via llama-quantize (Vulkan)
  3. #
  4. # 1) Download Q8_0 model from HuggingFace
  5. # 2) Download pre-built llama-quantize (Vulkan build)
  6. # 3) Quantize to target format
  7. # ============================================================================
  8.  
  9. import subprocess, sys, os, time
  10.  
  11. WORK_DIR = "/content/gguf_workspace"
  12. os.makedirs(WORK_DIR, exist_ok=True)
  13.  
  14. def install():
  15. for pkg in ["huggingface_hub[hf_xfer]"]:
  16. subprocess.check_call(
  17. [sys.executable, "-m", "pip", "install", "-q", pkg],
  18. stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
  19. )
  20. print("[OK] Dependencies installed.")
  21.  
  22. install()
  23.  
  24. from huggingface_hub import hf_hub_download
  25.  
  26. # ============================================================================
  27. # CONFIGURATION
  28. # ============================================================================
  29.  
  30. MODEL = {
  31. "repo": "your-username/your-repo-name",
  32. "file": "Qwen3.5-27B-Q8_0-merged.gguf",
  33. }
  34.  
  35. # Target quantization type
  36. # Options: Q4_K_M, Q4_K_S, Q5_K_M, Q5_K_S, Q6_K, Q3_K_M, Q3_K_L,
  37. # Q2_K, IQ4_XS, IQ3_XXS, IQ3_S, IQ2_XXS, IQ2_XS, IQ1_S, IQ1_M
  38. QUANT_TYPE = "Q4_K_M"
  39.  
  40. # llama.cpp Vulkan build URL
  41. LLAMA_URL = "https://github.com/ggml-org/llama.cpp/releases/download/b8468/llama-b8468-bin-ubuntu-vulkan-x64.tar.gz"
  42. LLAMA_TAR = "llama-b8468-bin-ubuntu-vulkan-x64.tar.gz"
  43.  
  44. # ============================================================================
  45. # UTILITY
  46. # ============================================================================
  47.  
  48. def disk_free_gb(p=WORK_DIR):
  49. st = os.statvfs(p)
  50. return (st.f_bavail * st.f_frsize) / 1073741824
  51.  
  52. def file_size_gb(p):
  53. return os.path.getsize(p) / 1073741824
  54.  
  55. def safe_remove(path, protect=None):
  56. try:
  57. a = os.path.abspath(path)
  58. if protect and a == os.path.abspath(protect):
  59. return
  60. if os.path.isfile(a):
  61. sz = file_size_gb(a)
  62. os.remove(a)
  63. print(f"[CLEANUP] Removed {os.path.basename(a)} ({sz:.2f} GB)")
  64. except OSError:
  65. pass
  66.  
  67. def check_gpu():
  68. """Check available GPU."""
  69. print("\n[GPU] Checking GPU...")
  70. try:
  71. result = subprocess.run(
  72. ["nvidia-smi", "--query-gpu=name,memory.total,driver_version",
  73. "--format=csv,noheader"],
  74. capture_output=True, text=True, timeout=10,
  75. )
  76. if result.returncode == 0:
  77. for line in result.stdout.strip().split('\n'):
  78. print(f"[GPU] {line.strip()}")
  79. return True
  80. except Exception:
  81. pass
  82. print("[GPU] nvidia-smi not available.")
  83. return False
  84.  
  85. # ============================================================================
  86. # DOWNLOAD MODEL
  87. # ============================================================================
  88.  
  89. def download_model(spec, label="Model"):
  90. print(f"\n[DL] {label}: {spec['repo']} / {spec['file']}")
  91. path = hf_hub_download(
  92. repo_id=spec["repo"],
  93. filename=spec["file"],
  94. local_dir=WORK_DIR,
  95. )
  96. print(f"[DL] {label}: {file_size_gb(path):.2f} GB -> {path}")
  97. return path
  98.  
  99. # ============================================================================
  100. # DOWNLOAD llama-quantize (Vulkan build)
  101. # ============================================================================
  102.  
  103. LLAMA_BIN_DIR = os.path.join(WORK_DIR, "llama-bin")
  104.  
  105. def get_llama_quantize():
  106. """Download pre-built llama-quantize (Vulkan build)."""
  107.  
  108. # Check if already available
  109. for root, dirs, files in os.walk(LLAMA_BIN_DIR):
  110. for f in files:
  111. if f == "llama-quantize":
  112. p = os.path.join(root, f)
  113. if os.access(p, os.X_OK):
  114. print(f"[BIN] Already available: {p}")
  115. return p
  116.  
  117. print(f"\n[BIN] Downloading Vulkan build...")
  118. print(f"[BIN] URL: {LLAMA_URL}")
  119. t0 = time.time()
  120.  
  121. tar_path = os.path.join(WORK_DIR, LLAMA_TAR)
  122.  
  123. # Download
  124. downloaded = False
  125. for cmd in [
  126. ["wget", "-q", "--show-progress", "-O", tar_path, LLAMA_URL],
  127. ["curl", "-sL", "-o", tar_path, LLAMA_URL],
  128. ]:
  129. try:
  130. print(f"[BIN] Trying: {cmd[0]}...")
  131. subprocess.check_call(cmd, timeout=300)
  132. if os.path.isfile(tar_path) and os.path.getsize(tar_path) > 10240:
  133. downloaded = True
  134. print(f"[BIN] Downloaded: {os.path.getsize(tar_path) / 1e6:.1f} MB")
  135. break
  136. except Exception as e:
  137. print(f"[BIN] {cmd[0]} failed: {e}")
  138. continue
  139.  
  140. if not downloaded:
  141. print("[BIN] ERROR: Failed to download llama.cpp binaries.")
  142. return None
  143.  
  144. # Extract
  145. os.makedirs(LLAMA_BIN_DIR, exist_ok=True)
  146.  
  147. for strip in ["--strip-components=1", None]:
  148. try:
  149. cmd = ["tar", "xzf", tar_path, "-C", LLAMA_BIN_DIR]
  150. if strip:
  151. cmd.append(strip)
  152. subprocess.check_call(cmd)
  153. break
  154. except subprocess.CalledProcessError:
  155. continue
  156.  
  157. # Remove archive
  158. if os.path.isfile(tar_path):
  159. os.remove(tar_path)
  160.  
  161. # Find llama-quantize
  162. for root, dirs, files in os.walk(LLAMA_BIN_DIR):
  163. for f in files:
  164. if f == "llama-quantize":
  165. binary = os.path.join(root, f)
  166. os.chmod(binary, 0o755)
  167. elapsed = time.time() - t0
  168. print(f"[BIN] Ready in {elapsed:.1f}s: {binary}")
  169. return binary
  170.  
  171. print("[BIN] ERROR: llama-quantize not found in archive.")
  172. print("[BIN] Archive contents:")
  173. for root, dirs, files in os.walk(LLAMA_BIN_DIR):
  174. for f in files:
  175. print(f" {os.path.relpath(os.path.join(root, f), LLAMA_BIN_DIR)}")
  176. return None
  177.  
  178. # ============================================================================
  179. # QUANTIZE
  180. # ============================================================================
  181.  
  182. def quantize(input_path, quant_type):
  183. """Run llama-quantize on the input GGUF file."""
  184.  
  185. print(f"\n{'='*75}")
  186. print(f"QUANTIZE: {os.path.basename(input_path)} -> {quant_type}")
  187. print(f"{'='*75}")
  188.  
  189. binary = get_llama_quantize()
  190. if binary is None:
  191. print("[QUANT] ERROR: llama-quantize not available.")
  192. return None
  193.  
  194. # Set LD_LIBRARY_PATH for shared libs next to the binary
  195. bin_dir = os.path.dirname(binary)
  196. env = os.environ.copy()
  197. ld_path = env.get("LD_LIBRARY_PATH", "")
  198. env["LD_LIBRARY_PATH"] = f"{bin_dir}:{ld_path}" if ld_path else bin_dir
  199.  
  200. # Build output filename
  201. base = os.path.splitext(os.path.basename(input_path))[0]
  202. for suffix in ["-Q8_0", "-Q4_0", "-Q4_K_M", "-Q4_K_S",
  203. "-Q5_K_M", "-Q5_K_S", "-Q6_K", "-Q3_K_M",
  204. "-Q3_K_L", "-Q3_K_S", "-Q2_K", "-F16", "-F32",
  205. "-BF16", "-IQ4_XS", "-IQ3_XXS", "-IQ3_S",
  206. "-IQ2_XXS", "-IQ2_XS", "-IQ1_S", "-IQ1_M"]:
  207. if base.endswith(suffix):
  208. base = base[:-len(suffix)]
  209. break
  210.  
  211. output_name = f"{base}-{quant_type}.gguf"
  212. output_path = os.path.join(WORK_DIR, output_name)
  213.  
  214. input_gb = file_size_gb(input_path)
  215. free_gb = disk_free_gb()
  216.  
  217. print(f"[QUANT] Input: {os.path.basename(input_path)} ({input_gb:.2f} GB)")
  218. print(f"[QUANT] Output: {output_name}")
  219. print(f"[QUANT] Quant type: {quant_type}")
  220. print(f"[QUANT] Disk free: {free_gb:.1f} GB")
  221.  
  222. if free_gb < input_gb * 0.3:
  223. print("[QUANT] ERROR: Not enough disk space.")
  224. return None
  225.  
  226. # Run llama-quantize
  227. cmd = [binary, "--allow-requantize", input_path, output_path, quant_type]
  228. print(f"[QUANT] Command: {' '.join(cmd)}")
  229. print(f"[QUANT] Running...\n")
  230.  
  231. t0 = time.time()
  232.  
  233. try:
  234. process = subprocess.Popen(
  235. cmd,
  236. stdout=subprocess.PIPE,
  237. stderr=subprocess.STDOUT,
  238. text=True,
  239. env=env,
  240. bufsize=1,
  241. )
  242.  
  243. for line in process.stdout:
  244. line = line.rstrip()
  245. if line:
  246. print(f" {line}")
  247.  
  248. process.wait()
  249.  
  250. if process.returncode != 0:
  251. print(f"\n[QUANT] ERROR: exit code {process.returncode}")
  252. return None
  253.  
  254. except Exception as e:
  255. print(f"[QUANT] ERROR: {e}")
  256. return None
  257.  
  258. if not os.path.isfile(output_path):
  259. print("[QUANT] ERROR: Output file not created.")
  260. return None
  261.  
  262. elapsed = time.time() - t0
  263. output_gb = file_size_gb(output_path)
  264. ratio = output_gb / input_gb * 100
  265.  
  266. print(f"\n[QUANT] Done in {elapsed:.0f}s ({elapsed/60:.1f} min)")
  267. print(f"[QUANT] {input_gb:.2f} GB -> {output_gb:.2f} GB ({ratio:.0f}%)")
  268. print(f"[QUANT] Speed: {input_gb / elapsed * 1024:.0f} MB/s")
  269.  
  270. return output_path
  271.  
  272. # ============================================================================
  273. # MAIN
  274. # ============================================================================
  275.  
  276. def main():
  277. print("=" * 75)
  278. print("GGUF QUANTIZE: Download Q8_0 -> llama-quantize (Vulkan)")
  279. print()
  280. print(f" Model: {MODEL['repo']} / {MODEL['file']}")
  281. print(f" Quant type: {QUANT_TYPE}")
  282. print(f" Build: {LLAMA_TAR}")
  283. print(f" WORK_DIR: {WORK_DIR}")
  284. print(f" Disk free: {disk_free_gb():.1f} GB")
  285. print("=" * 75)
  286.  
  287. check_gpu()
  288.  
  289. # STAGE 1: Download
  290. print("\n" + "=" * 75)
  291. print("STAGE 1: DOWNLOAD MODEL")
  292. print("=" * 75)
  293.  
  294. model_path = download_model(MODEL)
  295.  
  296. print(f"\n File: {os.path.basename(model_path)}")
  297. print(f" Size: {file_size_gb(model_path):.2f} GB")
  298. print(f" Disk free: {disk_free_gb():.1f} GB")
  299.  
  300. # STAGE 2: Quantize
  301. print("\n" + "=" * 75)
  302. print(f"STAGE 2: QUANTIZE -> {QUANT_TYPE}")
  303. print("=" * 75)
  304.  
  305. output_path = quantize(model_path, QUANT_TYPE)
  306.  
  307. if output_path and os.path.isfile(output_path):
  308. safe_remove(model_path, protect=output_path)
  309.  
  310. print("\n" + "=" * 75)
  311. print("DONE")
  312. print("=" * 75)
  313. print(f"\n[FS] {WORK_DIR}:")
  314. for fn in sorted(os.listdir(WORK_DIR)):
  315. fp = os.path.join(WORK_DIR, fn)
  316. if os.path.isfile(fp):
  317. print(f" {fn} ({file_size_gb(fp):.2f} GB)")
  318. print(f"\n Output: {output_path}")
  319. print(f" Size: {file_size_gb(output_path):.2f} GB")
  320. print(f" Disk free: {disk_free_gb():.1f} GB")
  321. else:
  322. print("\n[ERROR] Quantization failed.")
  323. print(f" Original kept: {model_path}")
  324. print(f" Size: {file_size_gb(model_path):.2f} GB")
  325.  
  326. main()
  327.  
Advertisement
Add Comment
Please, Sign In to add comment