Guest User

GS 3G CN Downloader

a guest
Nov 21st, 2020
61
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. import filetype
  2. import os
  3. import requests
  4. import shutil
  5. from threading import Thread
  6. from urllib.parse import urlparse, parse_qs
  7.  
  8. MAX_TRIES = 50
  9. POOL_SIZE = 10
  10. START_ID = "aaaaaa"
  11.  
  12. headers = {
  13.     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
  14.     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:83.0) Gecko/20100101 Firefox/83.0",
  15.     "Accept-Encoding": "gzip, deflate",
  16.     "Connection": "keep-alive"
  17. }
  18.  
  19.  
  20. def get_image(url, num_try=0):
  21.     if num_try > MAX_TRIES:
  22.         return None
  23.     t = requests.get(url, headers=headers)
  24.     if t.url != url:
  25.         parsed = urlparse(t.url)
  26.         parsed_q = parse_qs(parsed.query)
  27.         print(url)
  28.         print(t.url)
  29.         return parsed_q["u"][0]
  30.     else:
  31.         if t.text.startswith('{"state":"fail"'):
  32.             return "FAIL"
  33.         else:
  34.             return get_image(url, num_try=num_try + 1)
  35.  
  36.  
  37. ALLOWED_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
  38.  
  39.  
  40. def inc_char(c):
  41.     i = ALLOWED_CHARS.index(c)
  42.     if i < len(ALLOWED_CHARS) - 1:
  43.         return ALLOWED_CHARS[i + 1]
  44.     return ALLOWED_CHARS[0]
  45.  
  46.  
  47. def get_id(prev_id):
  48.     last = get_last_increment(prev_id)
  49.  
  50.     if last == -1:
  51.         return ""
  52.  
  53.     unchanged = prev_id[:last]
  54.     changed = prev_id[last:]
  55.     first_inc = inc_char(changed[0])
  56.     rest_count = len(changed) - 1
  57.  
  58.     return unchanged + first_inc + (ALLOWED_CHARS[0] * rest_count)
  59.  
  60.  
  61. def get_last_increment(str):
  62.     c = len(str) - 1
  63.     while c >= 0:
  64.         if str[c] != 'z':
  65.             return c
  66.         c -= 1
  67.  
  68.     return -1
  69.  
  70.  
  71. def build_url(id):
  72.     return f"http://gs.3g.cn/D/{id}/w"
  73.  
  74.  
  75. def download_data(id):
  76.     url = build_url(id)
  77.     print(f"ID: {id} URL: {url}")
  78.     try:
  79.         img = get_image(url)
  80.         if img is not None and img != "FAIL":
  81.             img_data = requests.get(img, stream=True)
  82.             img_data.decode_content = True
  83.             with open(f"output/{id}.bin", "wb") as output_file:
  84.                 shutil.copyfileobj(img_data.raw, output_file)
  85.             del img_data
  86.  
  87.             kind = filetype.guess(f"output/{id}.bin")
  88.             if kind is None:
  89.                 print(f"Unknown file type for {id}")
  90.             else:
  91.                 os.rename(f"output/{id}.bin", f"output/{id}.{kind.extension}")
  92.                 print(f"Saved file \"output/{id}.{kind.extension}\"")
  93.     except:
  94.         print(f"Skipped {id} due to many fails.")
  95.         pass
  96.     print()
  97.  
  98.  
  99. thread_pool = []
  100.  
  101. id = get_id(START_ID)
  102. while id != "":
  103.     while len(thread_pool) < POOL_SIZE:
  104.         t = Thread(target=download_data, args=(id,))
  105.         t.start()
  106.         thread_pool.append(t)
  107.         id = get_id(id)
  108.  
  109.     to_remove = []
  110.     for thread in thread_pool:
  111.         if not thread.is_alive():
  112.             to_remove.append(thread)
  113.  
  114.     for thread in to_remove:
  115.         thread_pool.remove(thread)
  116.         try:
  117.             thread.join()
  118.         except:
  119.             pass
  120.  
RAW Paste Data