MINEOGO

Untitled

Oct 24th, 2025
17
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 9.77 KB | None | 0 0
  1. #!/usr/bin/env python3
  2. """
  3. fast_downloader.py
  4. Multi-threaded downloader with progress, speed, ETA, and Android-style headers.
  5. Usage:
  6. python3 fast_downloader.py "https://example.com/file.apk" --out myfile.apk --threads 8
  7. """
  8.  
  9. import argparse
  10. import os
  11. import requests
  12. import threading
  13. import time
  14. from math import ceil
  15.  
  16. # Android-like headers
  17. ANDROID_HEADERS = {
  18. "User-Agent": "Dalvik/2.1.0 (Linux; U; Android 10; Pixel 3 Build/QP1A.190711.020)",
  19. "Accept": "*/*",
  20. "Accept-Encoding": "identity", # important for range downloads so we get raw bytes
  21. "Connection": "keep-alive",
  22. }
  23.  
  24. CHUNK = 8192 # 8 KB
  25.  
  26.  
  27. class ProgressTracker:
  28. def __init__(self, total_size):
  29. self.total = total_size
  30. self.lock = threading.Lock()
  31. self.downloaded = 0
  32. self.start_time = time.time()
  33. self.last_print = 0
  34.  
  35. def add(self, n):
  36. with self.lock:
  37. self.downloaded += n
  38.  
  39. def status(self):
  40. with self.lock:
  41. dl = self.downloaded
  42. now = time.time()
  43. elapsed = max(0.0001, now - self.start_time)
  44. speed = dl / elapsed # bytes/sec
  45. percent = (dl / self.total * 100) if self.total else 0
  46. remaining = (self.total - dl) if self.total else 0
  47. eta = (remaining / speed) if speed > 0 else float('inf')
  48. return dl, self.total, percent, speed, eta
  49.  
  50. def pretty(self):
  51. dl, total, percent, speed, eta = self.status()
  52. def fmt_bytes(b):
  53. for u in ['B','KB','MB','GB','TB']:
  54. if b < 1024.0:
  55. return f"{b:6.2f}{u}"
  56. b /= 1024.0
  57. return f"{b:.2f}PB"
  58. return f"{percent:6.2f}% {fmt_bytes(dl)}/{fmt_bytes(total)} {fmt_bytes(speed)}/s ETA {format_time(eta)}"
  59.  
  60.  
  61. def format_time(sec):
  62. if sec == float('inf'):
  63. return "--:--:--"
  64. sec = int(sec)
  65. h = sec // 3600
  66. m = (sec % 3600) // 60
  67. s = sec % 60
  68. return f"{h:02d}:{m:02d}:{s:02d}"
  69.  
  70.  
  71. def get_head(url, headers):
  72. return requests.head(url, headers=headers, allow_redirects=True, timeout=15)
  73.  
  74.  
  75. def supports_ranges(resp):
  76. ar = resp.headers.get("Accept-Ranges", "").lower()
  77. # some servers return 'bytes' or others; treat 'none' as unsupported
  78. return ar and ar != "none"
  79.  
  80.  
  81. def download_range(url, start, end, part_file, headers, tracker):
  82. # Range header expects inclusive bytes: start-end
  83. h = headers.copy()
  84. h["Range"] = f"bytes={start}-{end}"
  85. try:
  86. r = requests.get(url, headers=h, stream=True, timeout=60)
  87. r.raise_for_status()
  88. with open(part_file, "wb") as f:
  89. for chunk in r.iter_content(chunk_size=CHUNK):
  90. if not chunk:
  91. continue
  92. f.write(chunk)
  93. tracker.add(len(chunk))
  94. except Exception as e:
  95. # mark failure by deleting incomplete part
  96. if os.path.exists(part_file):
  97. # keep partial for resume? better to remove and let caller retry
  98. os.remove(part_file)
  99. raise
  100.  
  101.  
  102. def single_stream_download(url, out_path, headers, tracker):
  103. # single-threaded streaming download (resumable if server supports Range)
  104. mode = "wb"
  105. local_size = 0
  106. if os.path.exists(out_path):
  107. local_size = os.path.getsize(out_path)
  108. h = headers.copy()
  109. if local_size > 0:
  110. h["Range"] = f"bytes={local_size}-"
  111. mode = "ab"
  112. r = requests.get(url, headers=h, stream=True, timeout=60)
  113. r.raise_for_status()
  114. # If server returned full file despite Range, maybe restarting; adjust tracker
  115. content_length = r.headers.get("Content-Length")
  116. if content_length:
  117. try:
  118. tracker.total = local_size + int(content_length)
  119. except:
  120. pass
  121. with open(out_path, mode) as f:
  122. for chunk in r.iter_content(chunk_size=CHUNK):
  123. if not chunk:
  124. continue
  125. f.write(chunk)
  126. tracker.add(len(chunk))
  127.  
  128.  
  129. def merge_parts(part_files, out_path):
  130. with open(out_path, "wb") as outfile:
  131. for p in part_files:
  132. with open(p, "rb") as pf:
  133. while True:
  134. data = pf.read(CHUNK)
  135. if not data:
  136. break
  137. outfile.write(data)
  138. # optionally remove part files
  139. for p in part_files:
  140. os.remove(p)
  141.  
  142.  
  143. def multi_thread_download(url, out_path, headers, tracker, threads=8):
  144. # send a HEAD to get Content-Length and range support
  145. head = get_head(url, headers)
  146. head.raise_for_status()
  147. cl = head.headers.get("Content-Length")
  148. if cl is None:
  149. raise RuntimeError("Server did not provide Content-Length; cannot do multi-threaded download.")
  150. total = int(cl)
  151. tracker.total = total
  152.  
  153. if not supports_ranges(head):
  154. raise RuntimeError("Server does not support range requests.")
  155.  
  156. # split into ranges
  157. part_size = ceil(total / threads)
  158. part_files = []
  159. workers = []
  160. exceptions = []
  161. for i in range(threads):
  162. start = i * part_size
  163. end = min(total - 1, (i + 1) * part_size - 1)
  164. part_file = f"{out_path}.part{i}"
  165. part_files.append(part_file)
  166. # If part exists and has expected size, skip downloading
  167. if os.path.exists(part_file):
  168. existing = os.path.getsize(part_file)
  169. expected = end - start + 1
  170. if existing == expected:
  171. tracker.add(existing)
  172. continue
  173. else:
  174. # remove to restart
  175. os.remove(part_file)
  176.  
  177. t = threading.Thread(
  178. target=_thread_wrapper,
  179. args=(url, start, end, part_file, headers, tracker, exceptions),
  180. daemon=True,
  181. )
  182. workers.append(t)
  183.  
  184. # start all threads
  185. for w in workers:
  186. w.start()
  187.  
  188. last_print = 0
  189. try:
  190. while any(w.is_alive() for w in workers):
  191. now = time.time()
  192. if now - last_print >= 0.5:
  193. print("\r" + tracker.pretty(), end="", flush=True)
  194. last_print = now
  195. time.sleep(0.1)
  196. # join
  197. for w in workers:
  198. w.join()
  199. except KeyboardInterrupt:
  200. print("\nDownload interrupted by user. Partial parts remain.")
  201. raise
  202.  
  203. if exceptions:
  204. raise exceptions[0]
  205.  
  206. # final print
  207. print("\r" + tracker.pretty())
  208. merge_parts(part_files, out_path)
  209.  
  210.  
  211. def _thread_wrapper(url, start, end, part_file, headers, tracker, exceptions):
  212. try:
  213. download_range(url, start, end, part_file, headers, tracker)
  214. except Exception as e:
  215. exceptions.append(e)
  216.  
  217.  
  218. def human_readable(n):
  219. for unit in ['B','KB','MB','GB','TB']:
  220. if n < 1024:
  221. return f"{n:.2f}{unit}"
  222. n /= 1024
  223. return f"{n:.2f}PB"
  224.  
  225.  
  226. def main():
  227. parser = argparse.ArgumentParser(description="Fast downloader with progress/speed/ETA")
  228. parser.add_argument("url", help="URL to download")
  229. parser.add_argument("--out", "-o", help="Output filename (default: basename of url)", default=None)
  230. parser.add_argument("--threads", "-t", help="Number of threads for multi-conn download (default 8)", type=int, default=8)
  231. parser.add_argument("--no-android-headers", action="store_true", help="Don't use Android-style headers")
  232. args = parser.parse_args()
  233.  
  234. url = args.url
  235. out = args.out if args.out else os.path.basename(url.split("?")[0]) or "download.bin"
  236. threads = max(1, args.threads)
  237.  
  238. headers = {} if args.no_android_headers else ANDROID_HEADERS.copy()
  239.  
  240. # Probe server
  241. try:
  242. head = get_head(url, headers)
  243. head.raise_for_status()
  244. except Exception as e:
  245. print(f"HEAD request failed: {e}")
  246. print("Trying simple GET to see if URL is reachable...")
  247. try:
  248. r = requests.get(url, headers=headers, stream=True, timeout=20)
  249. r.raise_for_status()
  250. head = r
  251. except Exception as e2:
  252. print(f"Failed to reach URL: {e2}")
  253. return
  254.  
  255. content_length = head.headers.get("Content-Length")
  256. if content_length:
  257. total_size = int(content_length)
  258. else:
  259. total_size = None
  260.  
  261. tracker = ProgressTracker(total_size or 0)
  262.  
  263. try:
  264. if content_length and supports_ranges(head) and threads > 1:
  265. print(f"Server supports Range requests. Starting multi-threaded download using {threads} threads.")
  266. tracker.total = int(content_length)
  267. multi_thread_download(url, out, headers, tracker, threads=threads)
  268. else:
  269. if not content_length:
  270. print("No Content-Length; doing streaming download (single-threaded).")
  271. tracker.total = 0
  272. else:
  273. print("Range not supported or single-thread chosen; doing single-stream download.")
  274. tracker.total = int(content_length) if content_length else 0
  275. # show progress in background thread
  276. tprint = threading.Thread(target=_print_progress_loop, args=(tracker,), daemon=True)
  277. tprint.start()
  278. single_stream_download(url, out, headers, tracker)
  279. # final status
  280. print("\r" + tracker.pretty())
  281. print(f"Download complete: {out} ({human_readable(os.path.getsize(out))})")
  282. except Exception as e:
  283. print(f"\nError during download: {e}")
  284.  
  285.  
  286. def _print_progress_loop(tracker):
  287. last = 0
  288. while True:
  289. print("\r" + tracker.pretty(), end="", flush=True)
  290. time.sleep(0.5)
  291. # stop when downloaded >= total (if total known)
  292. if tracker.total and tracker.downloaded >= tracker.total:
  293. break
  294.  
  295.  
  296. if __name__ == "__main__":
  297. main()
Advertisement
Add Comment
Please, Sign In to add comment