Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # ============================================================================
- # GGUF Quantize: Download Q8_0 → Quantize via llama-quantize (Vulkan)
- #
- # 1) Download Q8_0 model from HuggingFace
- # 2) Download pre-built llama-quantize (Vulkan build)
- # 3) Quantize to target format
- # ============================================================================
- import subprocess, sys, os, time
- WORK_DIR = "/content/gguf_workspace"
- os.makedirs(WORK_DIR, exist_ok=True)
- def install():
- for pkg in ["huggingface_hub[hf_xfer]"]:
- subprocess.check_call(
- [sys.executable, "-m", "pip", "install", "-q", pkg],
- stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
- )
- print("[OK] Dependencies installed.")
- install()
- from huggingface_hub import hf_hub_download
- # ============================================================================
- # CONFIGURATION
- # ============================================================================
- MODEL = {
- "repo": "your-username/your-repo-name",
- "file": "Qwen3.5-27B-Q8_0-merged.gguf",
- }
- # Target quantization type
- # Options: Q4_K_M, Q4_K_S, Q5_K_M, Q5_K_S, Q6_K, Q3_K_M, Q3_K_L,
- # Q2_K, IQ4_XS, IQ3_XXS, IQ3_S, IQ2_XXS, IQ2_XS, IQ1_S, IQ1_M
- QUANT_TYPE = "Q4_K_M"
- # llama.cpp Vulkan build URL
- LLAMA_URL = "https://github.com/ggml-org/llama.cpp/releases/download/b8468/llama-b8468-bin-ubuntu-vulkan-x64.tar.gz"
- LLAMA_TAR = "llama-b8468-bin-ubuntu-vulkan-x64.tar.gz"
- # ============================================================================
- # UTILITY
- # ============================================================================
- def disk_free_gb(p=WORK_DIR):
- st = os.statvfs(p)
- return (st.f_bavail * st.f_frsize) / 1073741824
- def file_size_gb(p):
- return os.path.getsize(p) / 1073741824
- def safe_remove(path, protect=None):
- try:
- a = os.path.abspath(path)
- if protect and a == os.path.abspath(protect):
- return
- if os.path.isfile(a):
- sz = file_size_gb(a)
- os.remove(a)
- print(f"[CLEANUP] Removed {os.path.basename(a)} ({sz:.2f} GB)")
- except OSError:
- pass
- def check_gpu():
- """Check available GPU."""
- print("\n[GPU] Checking GPU...")
- try:
- result = subprocess.run(
- ["nvidia-smi", "--query-gpu=name,memory.total,driver_version",
- "--format=csv,noheader"],
- capture_output=True, text=True, timeout=10,
- )
- if result.returncode == 0:
- for line in result.stdout.strip().split('\n'):
- print(f"[GPU] {line.strip()}")
- return True
- except Exception:
- pass
- print("[GPU] nvidia-smi not available.")
- return False
- # ============================================================================
- # DOWNLOAD MODEL
- # ============================================================================
- def download_model(spec, label="Model"):
- print(f"\n[DL] {label}: {spec['repo']} / {spec['file']}")
- path = hf_hub_download(
- repo_id=spec["repo"],
- filename=spec["file"],
- local_dir=WORK_DIR,
- )
- print(f"[DL] {label}: {file_size_gb(path):.2f} GB -> {path}")
- return path
- # ============================================================================
- # DOWNLOAD llama-quantize (Vulkan build)
- # ============================================================================
- LLAMA_BIN_DIR = os.path.join(WORK_DIR, "llama-bin")
- def get_llama_quantize():
- """Download pre-built llama-quantize (Vulkan build)."""
- # Check if already available
- for root, dirs, files in os.walk(LLAMA_BIN_DIR):
- for f in files:
- if f == "llama-quantize":
- p = os.path.join(root, f)
- if os.access(p, os.X_OK):
- print(f"[BIN] Already available: {p}")
- return p
- print(f"\n[BIN] Downloading Vulkan build...")
- print(f"[BIN] URL: {LLAMA_URL}")
- t0 = time.time()
- tar_path = os.path.join(WORK_DIR, LLAMA_TAR)
- # Download
- downloaded = False
- for cmd in [
- ["wget", "-q", "--show-progress", "-O", tar_path, LLAMA_URL],
- ["curl", "-sL", "-o", tar_path, LLAMA_URL],
- ]:
- try:
- print(f"[BIN] Trying: {cmd[0]}...")
- subprocess.check_call(cmd, timeout=300)
- if os.path.isfile(tar_path) and os.path.getsize(tar_path) > 10240:
- downloaded = True
- print(f"[BIN] Downloaded: {os.path.getsize(tar_path) / 1e6:.1f} MB")
- break
- except Exception as e:
- print(f"[BIN] {cmd[0]} failed: {e}")
- continue
- if not downloaded:
- print("[BIN] ERROR: Failed to download llama.cpp binaries.")
- return None
- # Extract
- os.makedirs(LLAMA_BIN_DIR, exist_ok=True)
- for strip in ["--strip-components=1", None]:
- try:
- cmd = ["tar", "xzf", tar_path, "-C", LLAMA_BIN_DIR]
- if strip:
- cmd.append(strip)
- subprocess.check_call(cmd)
- break
- except subprocess.CalledProcessError:
- continue
- # Remove archive
- if os.path.isfile(tar_path):
- os.remove(tar_path)
- # Find llama-quantize
- for root, dirs, files in os.walk(LLAMA_BIN_DIR):
- for f in files:
- if f == "llama-quantize":
- binary = os.path.join(root, f)
- os.chmod(binary, 0o755)
- elapsed = time.time() - t0
- print(f"[BIN] Ready in {elapsed:.1f}s: {binary}")
- return binary
- print("[BIN] ERROR: llama-quantize not found in archive.")
- print("[BIN] Archive contents:")
- for root, dirs, files in os.walk(LLAMA_BIN_DIR):
- for f in files:
- print(f" {os.path.relpath(os.path.join(root, f), LLAMA_BIN_DIR)}")
- return None
- # ============================================================================
- # QUANTIZE
- # ============================================================================
- def quantize(input_path, quant_type):
- """Run llama-quantize on the input GGUF file."""
- print(f"\n{'='*75}")
- print(f"QUANTIZE: {os.path.basename(input_path)} -> {quant_type}")
- print(f"{'='*75}")
- binary = get_llama_quantize()
- if binary is None:
- print("[QUANT] ERROR: llama-quantize not available.")
- return None
- # Set LD_LIBRARY_PATH for shared libs next to the binary
- bin_dir = os.path.dirname(binary)
- env = os.environ.copy()
- ld_path = env.get("LD_LIBRARY_PATH", "")
- env["LD_LIBRARY_PATH"] = f"{bin_dir}:{ld_path}" if ld_path else bin_dir
- # Build output filename
- base = os.path.splitext(os.path.basename(input_path))[0]
- for suffix in ["-Q8_0", "-Q4_0", "-Q4_K_M", "-Q4_K_S",
- "-Q5_K_M", "-Q5_K_S", "-Q6_K", "-Q3_K_M",
- "-Q3_K_L", "-Q3_K_S", "-Q2_K", "-F16", "-F32",
- "-BF16", "-IQ4_XS", "-IQ3_XXS", "-IQ3_S",
- "-IQ2_XXS", "-IQ2_XS", "-IQ1_S", "-IQ1_M"]:
- if base.endswith(suffix):
- base = base[:-len(suffix)]
- break
- output_name = f"{base}-{quant_type}.gguf"
- output_path = os.path.join(WORK_DIR, output_name)
- input_gb = file_size_gb(input_path)
- free_gb = disk_free_gb()
- print(f"[QUANT] Input: {os.path.basename(input_path)} ({input_gb:.2f} GB)")
- print(f"[QUANT] Output: {output_name}")
- print(f"[QUANT] Quant type: {quant_type}")
- print(f"[QUANT] Disk free: {free_gb:.1f} GB")
- if free_gb < input_gb * 0.3:
- print("[QUANT] ERROR: Not enough disk space.")
- return None
- # Run llama-quantize
- cmd = [binary, "--allow-requantize", input_path, output_path, quant_type]
- print(f"[QUANT] Command: {' '.join(cmd)}")
- print(f"[QUANT] Running...\n")
- t0 = time.time()
- try:
- process = subprocess.Popen(
- cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- text=True,
- env=env,
- bufsize=1,
- )
- for line in process.stdout:
- line = line.rstrip()
- if line:
- print(f" {line}")
- process.wait()
- if process.returncode != 0:
- print(f"\n[QUANT] ERROR: exit code {process.returncode}")
- return None
- except Exception as e:
- print(f"[QUANT] ERROR: {e}")
- return None
- if not os.path.isfile(output_path):
- print("[QUANT] ERROR: Output file not created.")
- return None
- elapsed = time.time() - t0
- output_gb = file_size_gb(output_path)
- ratio = output_gb / input_gb * 100
- print(f"\n[QUANT] Done in {elapsed:.0f}s ({elapsed/60:.1f} min)")
- print(f"[QUANT] {input_gb:.2f} GB -> {output_gb:.2f} GB ({ratio:.0f}%)")
- print(f"[QUANT] Speed: {input_gb / elapsed * 1024:.0f} MB/s")
- return output_path
- # ============================================================================
- # MAIN
- # ============================================================================
- def main():
- print("=" * 75)
- print("GGUF QUANTIZE: Download Q8_0 -> llama-quantize (Vulkan)")
- print()
- print(f" Model: {MODEL['repo']} / {MODEL['file']}")
- print(f" Quant type: {QUANT_TYPE}")
- print(f" Build: {LLAMA_TAR}")
- print(f" WORK_DIR: {WORK_DIR}")
- print(f" Disk free: {disk_free_gb():.1f} GB")
- print("=" * 75)
- check_gpu()
- # STAGE 1: Download
- print("\n" + "=" * 75)
- print("STAGE 1: DOWNLOAD MODEL")
- print("=" * 75)
- model_path = download_model(MODEL)
- print(f"\n File: {os.path.basename(model_path)}")
- print(f" Size: {file_size_gb(model_path):.2f} GB")
- print(f" Disk free: {disk_free_gb():.1f} GB")
- # STAGE 2: Quantize
- print("\n" + "=" * 75)
- print(f"STAGE 2: QUANTIZE -> {QUANT_TYPE}")
- print("=" * 75)
- output_path = quantize(model_path, QUANT_TYPE)
- if output_path and os.path.isfile(output_path):
- safe_remove(model_path, protect=output_path)
- print("\n" + "=" * 75)
- print("DONE")
- print("=" * 75)
- print(f"\n[FS] {WORK_DIR}:")
- for fn in sorted(os.listdir(WORK_DIR)):
- fp = os.path.join(WORK_DIR, fn)
- if os.path.isfile(fp):
- print(f" {fn} ({file_size_gb(fp):.2f} GB)")
- print(f"\n Output: {output_path}")
- print(f" Size: {file_size_gb(output_path):.2f} GB")
- print(f" Disk free: {disk_free_gb():.1f} GB")
- else:
- print("\n[ERROR] Quantization failed.")
- print(f" Original kept: {model_path}")
- print(f" Size: {file_size_gb(model_path):.2f} GB")
- main()
Advertisement
Add Comment
Please, Sign In to add comment