SVXX

GPU Orchestrator Script

May 20th, 2026 (edited)
100
0
Never
1
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 17.43 KB | None | 0 0
  1. """
  2. GPU-aware command queue runner.
  3.  
  4. Reads a file of shell commands (one per line) and dispatches them to GPUs as
  5. memory becomes available. CUDA_VISIBLE_DEVICES= prefixes and trailing & are
  6. stripped from each command; the assigned GPU is injected via the subprocess
  7. environment instead.
  8.  
  9. Blank lines and lines starting with # are ignored. The script is resumable:
  10. state is written to a JSON file next to the commands file, so re-running the
  11. same invocation picks up where it left off.
  12.  
  13. Usage:
  14.    python cmd_queue.py <commands_file> [options]
  15.  
  16. Options:
  17.    --min-free-mb INT    Minimum free GPU memory in MiB to consider a GPU
  18.                         available (default: 10000)
  19.    --poll-interval INT  Seconds between GPU polls (default: 60)
  20.    --max-retries INT    Maximum retries for a failed command (default: 3)
  21.    --max-jobs INT       Maximum concurrent jobs (default: unlimited)
  22.    --state-file PATH    Explicit path for state JSON (default: auto-derived
  23.                         from commands file name + content hash)
  24.  
  25. Example:
  26.    python cmd_queue.py my_runs.txt --min-free-mb 12000
  27. """
  28.  
  29. # ruff: noqa: E402
  30. from dotenv import load_dotenv
  31.  
  32. load_dotenv()
  33.  
  34. import argparse
  35. import hashlib
  36. import json
  37. import os
  38. import re
  39. import signal
  40. import subprocess
  41. import time
  42. from pathlib import Path
  43.  
  44. from loguru import logger
  45.  
  46.  
  47. # ---------------------------------------------------------------------------
  48. # Command parsing
  49. # ---------------------------------------------------------------------------
  50.  
  51. _CUDA_PREFIX_RE = re.compile(r"^(CUDA_VISIBLE_DEVICES=\S+\s+)+")
  52.  
  53.  
  54. def _parse_commands(path: Path) -> list[str]:
  55.     """Read commands file, skip blanks and comments, return cleaned commands."""
  56.     raw_lines = path.read_text().splitlines()
  57.     commands: list[str] = []
  58.     for line in raw_lines:
  59.         stripped = line.strip()
  60.         if not stripped or stripped.startswith("#"):
  61.             continue
  62.         # Strip CUDA_VISIBLE_DEVICES=X prefix(es)
  63.         stripped = _CUDA_PREFIX_RE.sub("", stripped)
  64.         # Strip trailing &
  65.         stripped = stripped.rstrip("& ").rstrip()
  66.         if stripped:
  67.             commands.append(stripped)
  68.     return commands
  69.  
  70.  
  71. # ---------------------------------------------------------------------------
  72. # Orchestrator
  73. # ---------------------------------------------------------------------------
  74.  
  75. class CmdQueueOrchestrator:
  76.     def __init__(
  77.         self,
  78.         commands: list[str],
  79.         state_file: Path,
  80.         min_free_mb: int,
  81.         poll_interval: int,
  82.         max_retries: int,
  83.         max_jobs: int | None = None,
  84.         no_gpu_manage: bool = False,
  85.         retry_failed: bool = False,
  86.         dispatch_delay: float = 0,
  87.         name: str | None = None,
  88.         allowed_gpus: set[int] | None = None,
  89.         one_job_per_gpu: bool = False,
  90.     ):
  91.         self.commands = commands
  92.         self.state_file = state_file
  93.         self.min_free_mb = min_free_mb
  94.         self.poll_interval = poll_interval
  95.         self.max_retries = max_retries
  96.         self.max_jobs = max_jobs
  97.         self.no_gpu_manage = no_gpu_manage
  98.         self.retry_failed = retry_failed
  99.         self.dispatch_delay = dispatch_delay
  100.         self.name = name
  101.         self.allowed_gpus = allowed_gpus
  102.         self.one_job_per_gpu = one_job_per_gpu
  103.         # run_key -> Popen, for live processes in this session
  104.         self._processes: dict[str, subprocess.Popen] = {}
  105.  
  106.     def _log(self, level: str, msg: str) -> None:
  107.         prefix = f"[{self.name}] " if self.name else ""
  108.         getattr(logger, level)(f"{prefix}{msg}")
  109.  
  110.     # ------------------------------------------------------------------
  111.     # State helpers
  112.     # ------------------------------------------------------------------
  113.  
  114.     @staticmethod
  115.     def _run_key(idx: int) -> str:
  116.         return f"cmd_{idx:04d}"
  117.  
  118.     def _load_state(self) -> dict:
  119.         if self.state_file.exists():
  120.             with open(self.state_file) as f:
  121.                 state = json.load(f)
  122.             self._log("info", f"Resumed state from {self.state_file}")
  123.             for key, info in state.items():
  124.                 if info["status"] == "running":
  125.                     self._log("warning",
  126.                         f"{key}: was 'running' in state file with no live process"
  127.                         " — marking pending"
  128.                     )
  129.                     info["status"] = "pending"
  130.                 elif info["status"] == "failed" and self.retry_failed:
  131.                     self._log("info", f"{key}: retrying previously failed command")
  132.                     info["status"] = "pending"
  133.                     info["retries"] = 0
  134.             return state
  135.  
  136.         state = {
  137.             self._run_key(idx): {
  138.                 "cmd": cmd,
  139.                 "status": "pending",
  140.                 "retries": 0,
  141.             }
  142.             for idx, cmd in enumerate(self.commands)
  143.         }
  144.         self._save_state(state)
  145.         return state
  146.  
  147.     def _save_state(self, state: dict) -> None:
  148.         with open(self.state_file, "w") as f:
  149.             json.dump(state, f, indent=2)
  150.  
  151.     # ------------------------------------------------------------------
  152.     # GPU helpers
  153.     # ------------------------------------------------------------------
  154.  
  155.     def _available_gpus(self) -> list[tuple[int, int]]:
  156.         """Return (gpu_idx, free_mib) pairs for GPUs with >= min_free_mb free,
  157.        sorted by free memory descending."""
  158.         try:
  159.             result = subprocess.run(
  160.                 [
  161.                     "nvidia-smi",
  162.                     "--query-gpu=index,memory.free",
  163.                     "--format=csv,noheader,nounits",
  164.                 ],
  165.                 capture_output=True,
  166.                 text=True,
  167.                 check=True,
  168.             )
  169.         except (FileNotFoundError, subprocess.CalledProcessError) as e:
  170.             self._log("warning", f"nvidia-smi failed: {e}")
  171.             return []
  172.  
  173.         candidates: list[tuple[int, int]] = []
  174.         for line in result.stdout.strip().splitlines():
  175.             parts = line.split(",")
  176.             if len(parts) != 2:
  177.                 continue
  178.             idx, free_mib = int(parts[0].strip()), int(parts[1].strip())
  179.             if self.allowed_gpus is not None and idx not in self.allowed_gpus:
  180.                 continue
  181.             if free_mib >= self.min_free_mb:
  182.                 candidates.append((idx, free_mib))
  183.  
  184.         candidates.sort(key=lambda x: -x[1])  # most free first
  185.         return candidates
  186.  
  187.     # ------------------------------------------------------------------
  188.     # Dispatch
  189.     # ------------------------------------------------------------------
  190.  
  191.     def _dispatch(self, run_key: str, cmd: str, gpu_id: int | None) -> subprocess.Popen:
  192.         if gpu_id is not None:
  193.             env = {**os.environ, "CUDA_VISIBLE_DEVICES": str(gpu_id)}
  194.             gpu_label = f"GPU {gpu_id}"
  195.         else:
  196.             env = None
  197.             gpu_label = "inherited GPU"
  198.         proc = subprocess.Popen(cmd, shell=True, env=env, start_new_session=True)
  199.         self._log("info", f"Dispatched {run_key} → {gpu_label} (pid={proc.pid})\n  cmd: {cmd}")
  200.         return proc
  201.  
  202.     # ------------------------------------------------------------------
  203.     # Main loop
  204.     # ------------------------------------------------------------------
  205.  
  206.     def _check_processes(self, state: dict) -> dict:
  207.         for run_key, proc in list(self._processes.items()):
  208.             ret = proc.poll()
  209.             if ret is None:
  210.                 continue
  211.  
  212.             del self._processes[run_key]
  213.  
  214.             if ret == 0:
  215.                 state[run_key]["status"] = "done"
  216.                 self._log("info", f"{run_key} completed successfully")
  217.             else:
  218.                 state[run_key]["retries"] += 1
  219.                 retries = state[run_key]["retries"]
  220.                 if retries > self.max_retries:
  221.                     state[run_key]["status"] = "failed"
  222.                     self._log("error",
  223.                         f"{run_key} failed permanently after {retries} attempt(s) (exit {ret})"
  224.                     )
  225.                 else:
  226.                     state[run_key]["status"] = "pending"
  227.                     self._log("warning",
  228.                         f"{run_key} failed (exit {ret}), queued for retry"
  229.                         f" {retries}/{self.max_retries}"
  230.                     )
  231.         return state
  232.  
  233.     def run(self) -> None:
  234.         state = self._load_state()
  235.  
  236.         try:
  237.             while True:
  238.                 state = self._check_processes(state)
  239.                 self._save_state(state)
  240.  
  241.                 by_status: dict[str, list[str]] = {
  242.                     "pending": [], "running": [], "done": [], "failed": [],
  243.                 }
  244.                 for key, info in state.items():
  245.                     by_status.setdefault(info["status"], []).append(key)
  246.  
  247.                 self._log("info",
  248.                     f"Status — done: {len(by_status['done'])}, "
  249.                     f"running: {len(by_status['running'])}, "
  250.                     f"pending: {len(by_status['pending'])}, "
  251.                     f"failed: {len(by_status['failed'])}"
  252.                 )
  253.  
  254.                 if not by_status["pending"] and not by_status["running"]:
  255.                     self._log("info", "All commands finished.")
  256.                     if by_status["failed"]:
  257.                         self._log("warning", f"Failed commands: {by_status['failed']}")
  258.                     break
  259.  
  260.                 if by_status["pending"]:
  261.                     if self.max_jobs is not None and len(by_status["running"]) >= self.max_jobs:
  262.                         self._log("info",
  263.                             f"Max jobs ({self.max_jobs}) running. Waiting {self.poll_interval}s..."
  264.                         )
  265.                     elif self.no_gpu_manage:
  266.                         for run_key in by_status["pending"]:
  267.                             if self.max_jobs is not None and len(self._processes) >= self.max_jobs:
  268.                                 break
  269.                             cmd = state[run_key]["cmd"]
  270.                             proc = self._dispatch(run_key, cmd, gpu_id=None)
  271.                             state[run_key].update({"status": "running", "pid": proc.pid})
  272.                             self._processes[run_key] = proc
  273.                             self._save_state(state)
  274.                             if self.dispatch_delay > 0:
  275.                                 time.sleep(self.dispatch_delay)
  276.                     else:
  277.                         # Subtract min_free_mb per already-dispatched run to avoid
  278.                         # over-committing before processes actually allocate memory.
  279.                         free_mib: dict[int, int] = dict(self._available_gpus())
  280.  
  281.                         if self.one_job_per_gpu:
  282.                             busy = {
  283.                                 info.get("gpu")
  284.                                 for info in state.values()
  285.                                 if info["status"] == "running" and "gpu" in info
  286.                             }
  287.                             free_mib = {k: v for k, v in free_mib.items() if k not in busy}
  288.  
  289.                         if not free_mib:
  290.                             self._log("info",
  291.                                 f"No GPUs with >= {self.min_free_mb} MiB free right now. "
  292.                                 f"Waiting {self.poll_interval}s..."
  293.                             )
  294.                         else:
  295.                             for run_key in by_status["pending"]:
  296.                                 if self.max_jobs is not None and len(self._processes) >= self.max_jobs:
  297.                                     break
  298.                                 eligible = [
  299.                                     (idx, mib) for idx, mib in free_mib.items()
  300.                                     if mib >= self.min_free_mb
  301.                                 ]
  302.                                 if not eligible:
  303.                                     break
  304.                                 gpu_id, _ = max(eligible, key=lambda x: x[1])
  305.                                 if self.one_job_per_gpu:
  306.                                     del free_mib[gpu_id]
  307.                                 else:
  308.                                     free_mib[gpu_id] -= self.min_free_mb
  309.                                 cmd = state[run_key]["cmd"]
  310.                                 proc = self._dispatch(run_key, cmd, gpu_id)
  311.                                 state[run_key].update(
  312.                                     {"status": "running", "gpu": gpu_id, "pid": proc.pid}
  313.                                 )
  314.                                 self._processes[run_key] = proc
  315.                                 self._save_state(state)
  316.                                 if self.dispatch_delay > 0:
  317.                                     time.sleep(self.dispatch_delay)
  318.  
  319.                 time.sleep(self.poll_interval)
  320.  
  321.         except KeyboardInterrupt:
  322.             self._log("warning", "Interrupted — terminating running subprocesses...")
  323.             for run_key, proc in self._processes.items():
  324.                 os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
  325.                 state[run_key]["status"] = "pending"
  326.             self._save_state(state)
  327.             self._log("info", f"State saved to {self.state_file}. Re-run to resume.")
  328.  
  329.  
  330. # ---------------------------------------------------------------------------
  331. # Entry point
  332. # ---------------------------------------------------------------------------
  333.  
  334. def main() -> None:
  335.     parser = argparse.ArgumentParser(
  336.         description="GPU-aware command queue runner.",
  337.         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
  338.     )
  339.     parser.add_argument("commands_file", type=Path, help="File with one shell command per line")
  340.     parser.add_argument("--name", type=str, default=None,
  341.                         help="Human-readable identifier for this queue (shown in all log output)")
  342.     parser.add_argument("--min-free-mb", type=int, default=10000,
  343.                         help="Minimum free GPU memory (MiB) to dispatch a command")
  344.     parser.add_argument("--poll-interval", type=int, default=60,
  345.                         help="Seconds between GPU polls")
  346.     parser.add_argument("--max-retries", type=int, default=3,
  347.                         help="Max retries for a failed command")
  348.     parser.add_argument("--max-jobs", type=int, default=None,
  349.                         help="Maximum number of concurrently running jobs (default: unlimited)")
  350.     parser.add_argument("--gpus", type=str, default=None,
  351.                         help="Comma-separated GPU indices to use (e.g. 0,1,2,3). "
  352.                              "By default all GPUs are eligible.")
  353.     parser.add_argument("--one-job-per-gpu", action="store_true",
  354.                         help="Allow at most one running job per GPU (instead of packing by memory).")
  355.     parser.add_argument("--no-gpu-manage", action="store_true",
  356.                         help="Skip nvidia-smi and CUDA_VISIBLE_DEVICES injection (use for SLURM/MIG)")
  357.     parser.add_argument("--retry-failed", action="store_true",
  358.                         help="Re-queue all failed commands from a previous run")
  359.     parser.add_argument("--dispatch-delay", type=float, default=0,
  360.                         help="Seconds to sleep between dispatching each job (to avoid API rate limits)")
  361.     parser.add_argument("--state-file", type=Path, default=None,
  362.                         help="Exact path for the state JSON file (auto-derived if omitted)")
  363.     parser.add_argument("--state-dir", type=Path, default=None,
  364.                         help="Directory to write the auto-named state file into (created if needed)")
  365.     args = parser.parse_args()
  366.  
  367.     if args.state_file and args.state_dir:
  368.         parser.error("--state-file and --state-dir are mutually exclusive")
  369.  
  370.     commands_file: Path = args.commands_file.resolve()
  371.     if not commands_file.exists():
  372.         parser.error(f"Commands file not found: {commands_file}")
  373.  
  374.     commands = _parse_commands(commands_file)
  375.     if not commands:
  376.         logger.warning("No commands found in file. Exiting.")
  377.         return
  378.  
  379.     name: str | None = args.name
  380.     prefix = f"[{name}] " if name else ""
  381.     logger.info(f"{prefix}Loaded {len(commands)} command(s) from {commands_file}")
  382.  
  383.     if args.state_file is not None:
  384.         state_file = args.state_file
  385.     else:
  386.         content_hash = hashlib.sha256(commands_file.read_bytes()).hexdigest()[:8]
  387.         stem = name if name else commands_file.stem
  388.         state_dir = args.state_dir.resolve() if args.state_dir else commands_file.parent
  389.         state_dir.mkdir(parents=True, exist_ok=True)
  390.         state_file = state_dir / f"{stem}_state_{content_hash}.json"
  391.  
  392.     logger.info(f"{prefix}State file: {state_file}")
  393.  
  394.     allowed_gpus = (
  395.         {int(x.strip()) for x in args.gpus.split(",")} if args.gpus else None
  396.     )
  397.     if allowed_gpus is not None:
  398.         logger.info(f"{prefix}Restricting to GPUs: {sorted(allowed_gpus)}")
  399.  
  400.     orchestrator = CmdQueueOrchestrator(
  401.         commands=commands,
  402.         state_file=state_file,
  403.         min_free_mb=args.min_free_mb,
  404.         poll_interval=args.poll_interval,
  405.         max_retries=args.max_retries,
  406.         max_jobs=args.max_jobs,
  407.         no_gpu_manage=args.no_gpu_manage,
  408.         retry_failed=args.retry_failed,
  409.         dispatch_delay=args.dispatch_delay,
  410.         name=name,
  411.         allowed_gpus=allowed_gpus,
  412.         one_job_per_gpu=args.one_job_per_gpu,
  413.     )
  414.     orchestrator.run()
  415.  
  416.  
  417. if __name__ == "__main__":
  418.     main()
  419.  
Advertisement
Comments
  • User was banned
Add Comment
Please, Sign In to add comment