API tools faq
paste
Advertisement
Guest User

VR1.0.5.0

a guest
Nov 14th, 2022
6,339
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 32.75 KB | Help | 0 0
raw download clone embed print report
  1. import datetime
  2. import hashlib
  3. import os
  4. import random
  5. import re
  6. from datetime import timedelta
  7. import grequests
  8. import requests
  9. from bs4 import BeautifulSoup
  10. from moviepy.editor import concatenate_videoclips, VideoFileClip
  11. from natsort import natsorted
  12. """
  13. * Created By: ItIckeYd ver 1.0.5.0 ~20221110
  14. """
  15. domains = ["https://vod-secure.twitch.tv/",
  16.            "https://vod-metro.twitch.tv/",
  17.            "https://vod-pop-secure.twitch.tv/",
  18.            "https://d2e2de1etea730.cloudfront.net/",
  19.            "https://dqrpb9wgowsf5.cloudfront.net/",
  20.            "https://ds0h3roq6wcgc.cloudfront.net/",
  21.            "https://d2nvs31859zcd8.cloudfront.net/",
  22.            "https://d2aba1wr3818hz.cloudfront.net/",
  23.            "https://d3c27h4odz752x.cloudfront.net/",
  24.            "https://dgeft87wbj63p.cloudfront.net/",
  25.            "https://d1m7jfoe9zdc1j.cloudfront.net/",
  26.            "https://d3vd9lfkzbru3h.cloudfront.net/",
  27.            "https://d2vjef5jvl6bfs.cloudfront.net/",
  28.            "https://d1ymi26ma8va5x.cloudfront.net/",
  29.            "https://d1mhjrowxxagfy.cloudfront.net/",
  30.            "https://ddacn6pr5v0tl.cloudfront.net/",
  31.            "https://d3aqoihi2n8ty8.cloudfront.net/"]
  32.  
  33. user_agents = ["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
  34.                "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
  35.                "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
  36.                "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
  37.                "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36",
  38.                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:103.0) Gecko/20100101 Firefox/103.0",
  39.                "Mozilla/5.0 (Macintosh; Intel Mac OS X 12.5; rv:103.0) Gecko/20100101 Firefox/103.0",
  40.                "Mozilla/5.0 (X11; Linux i686; rv:103.0) Gecko/20100101 Firefox/103.0",
  41.                "Mozilla/5.0 (Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0",
  42.                "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:103.0) Gecko/20100101 Firefox/103.0",
  43.                "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0",
  44.                "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:103.0) Gecko/20100101 Firefox/103.0",
  45.                "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:102.0) Gecko/20100101 Firefox/102.0",
  46.                "Mozilla/5.0 (Macintosh; Intel Mac OS X 12.5; rv:102.0) Gecko/20100101 Firefox/102.0",
  47.                "Mozilla/5.0 (X11; Linux i686; rv:102.0) Gecko/20100101 Firefox/102.0",
  48.                "Mozilla/5.0 (Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0",
  49.                "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:102.0) Gecko/20100101 Firefox/102.0",
  50.                "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0",
  51.                "Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0",
  52.                "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.6 Safari/605.1.15",
  53.                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36 Edg/103.0.1264.77",
  54.                "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.36 Edg/103.0.1264.77",
  55.                'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36']
  56.  
  57.  
  58. def return_main_menu():
  59.     print("WELCOME TO VOD RECOVERY" + "\n")
  60.     menu = "1) Recover Vod" + "\n" + "2) Recover Clips" + "\n" + "3) Unmute an M3U8 file" + "\n" + "4) Check M3U8 Segments" + "\n" + "5) Generate M3U8 file (ONLY includes valid segments)" + "\n" + "6) Download M3U8 (.MP4 extension)" + "\n" + "7) Exit" + "\n"
  61.     print(menu)
  62.  
  63.  
  64. def get_default_directory():
  65.     return os.path.expanduser("~/Documents/")
  66.  
  67.  
  68. def generate_log_filename(streamer, vod_id):
  69.     log_filename = os.path.join(get_default_directory(), streamer + "_" + vod_id + "_log.txt")
  70.     return log_filename
  71.  
  72.  
  73. def generate_vod_filename(streamer, vod_id):
  74.     vod_filename = os.path.join(get_default_directory(), "VodRecovery_" + streamer + "_" + vod_id + ".m3u8")
  75.     return vod_filename
  76.  
  77.  
  78. def generate_website_links(streamer, vod_id):
  79.     website_list = ["https://sullygnome.com/channel/" + streamer + "/stream/" + vod_id,
  80.                     "https://twitchtracker.com/" + streamer + "/streams/" + vod_id,
  81.                     "https://streamscharts.com/channels/" + streamer + "/streams/" + vod_id]
  82.  
  83.     return website_list
  84.  
  85.  
  86. def return_header():
  87.     header = {
  88.         'user-agent': f'{random.choice(user_agents)}'
  89.     }
  90.     return header
  91.  
  92.  
  93. def remove_file(file_path):
  94.     if os.path.exists(file_path):
  95.         return os.remove(file_path)
  96.  
  97.  
  98. def format_timestamp(timestamp):
  99.     formatted_date = datetime.datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S")
  100.     return formatted_date
  101.  
  102.  
  103. def get_vod_age(timestamp):
  104.     vod_age = datetime.datetime.today() - format_timestamp(timestamp)
  105.     if vod_age.days <= 0:
  106.         return 0
  107.     else:
  108.         return vod_age.days
  109.  
  110.  
  111. def is_vod_muted(url):
  112.     return bool("unmuted" in requests.get(url).text)
  113.  
  114.  
  115. def get_duration(hours, minutes):
  116.     return (int(hours) * 60) + int(minutes)
  117.  
  118.  
  119. def get_reps(duration):
  120.     reps = ((duration * 60) + 2000)
  121.     return reps
  122.  
  123.  
  124. def get_clip_format(vod_id, reps):
  125.     default_clip_list = ["https://clips-media-assets2.twitch.tv/" + vod_id + "-offset-" + str(i) + ".mp4" for i in
  126.                          range(reps) if i % 2 == 0]
  127.  
  128.     alternate_clip_list = ["https://clips-media-assets2.twitch.tv/vod-" + vod_id + "-offset-" + str(i) + ".mp4" for i in
  129.                            range(reps) if i % 2 == 0]
  130.  
  131.     legacy_clip_list = [
  132.         "https://clips-media-assets2.twitch.tv/" + vod_id + "-index-" + "%010g" % (int('000000000') + i) + ".mp4" for i
  133.         in range(reps)]
  134.  
  135.     clip_format_dict = {}
  136.  
  137.     clip_format_dict.update({"1": default_clip_list})
  138.     clip_format_dict.update({"2": alternate_clip_list})
  139.     clip_format_dict.update({"3": legacy_clip_list})
  140.  
  141.     return clip_format_dict
  142.  
  143.  
  144. def get_all_clip_urls(clip_dict, clip_format):
  145.     full_url_list = []
  146.     for key, value in clip_dict.items():
  147.         if key in clip_format:
  148.             full_url_list += value
  149.     return full_url_list
  150.  
  151.  
  152. def return_username(url):
  153.     indices = [i.start() for i in re.finditer('_', url)]
  154.     username = url[indices[0] + 1:indices[-2]]
  155.     return username
  156.  
  157.  
  158. def return_vod_id(url):
  159.     indices = [i.start() for i in re.finditer('_', url)]
  160.     vod_id = url[indices[0] + len(return_username(url)) + 2:indices[-1]]
  161.     return vod_id
  162.  
  163.  
  164. def return_file_contents(streamer, vod_id):
  165.     with open(generate_log_filename(streamer, vod_id)) as f:
  166.         content = f.readlines()
  167.         content = [x.strip() for x in content]
  168.     return content
  169.  
  170.  
  171. def get_vod_urls(streamer, vod_id, timestamp):
  172.     vod_url_list, valid_vod_url_list = [], []
  173.     for seconds in range(60):
  174.         epoch_timestamp = ((format_timestamp(timestamp) + timedelta(seconds=seconds)) - datetime.datetime(1970, 1,
  175.                                                                                                           1)).total_seconds()
  176.         base_url = streamer + "_" + vod_id + "_" + str(int(epoch_timestamp))
  177.         hashed_base_url = str(hashlib.sha1(base_url.encode('utf-8')).hexdigest())[:20]
  178.         for domain in domains:
  179.             vod_url_list.append(domain + hashed_base_url + "_" + base_url + "/chunked/index-dvr.m3u8")
  180.     request_session = requests.Session()
  181.     rs = [grequests.head(u, session=request_session) for u in vod_url_list]
  182.     for result in grequests.imap(rs, size=100):
  183.         if result.status_code == 200:
  184.             valid_vod_url_list.append(result.url)
  185.     return valid_vod_url_list
  186.  
  187.  
  188. def parse_duration_streamscharts(tracker_url):
  189.     for _ in range(10):
  190.         response = requests.get(tracker_url, headers=return_header(), allow_redirects=False)
  191.         if response.status_code == 200:
  192.             bs = BeautifulSoup(response.content, 'html.parser')
  193.             streamscharts_duration = bs.find_all('div', {'class': 'text-xs font-bold'})[3].text.strip().replace("h", "").replace("m", "").split(" ")
  194.             return get_duration(int(streamscharts_duration[0]), int(streamscharts_duration[1]))
  195.  
  196.  
  197. def parse_duration_twitchtracker(tracker_url):
  198.     response = requests.get(tracker_url, headers=return_header(), allow_redirects=False)
  199.     if response.status_code == 200:
  200.         bs = BeautifulSoup(response.content, 'html.parser')
  201.         twitchtracker_duration = bs.find_all('div', {'class': 'g-x-s-value'})[0].text
  202.         return twitchtracker_duration
  203.  
  204.  
  205. def parse_duration_sullygnome(tracker_url):
  206.     response = requests.get(tracker_url, headers=return_header(), allow_redirects=False)
  207.     if response.status_code == 200:
  208.         bs = BeautifulSoup(response.content, 'html.parser')
  209.         sullygnome_duration = bs.find_all('div', {'class': 'MiddleSubHeaderItemValue'})[7].text.strip().replace("hours", "").replace("minutes", "").split(",")
  210.         return get_duration(int(sullygnome_duration[0]), int(sullygnome_duration[1]))
  211.  
  212.  
  213. def parse_datetime_streamscharts(tracker_url):
  214.     for _ in range(10):
  215.         response = requests.get(tracker_url, headers=return_header(), allow_redirects=False)
  216.         if response.status_code == 200:
  217.             bs = BeautifulSoup(response.content, 'html.parser')
  218.             streamscharts_datetime = bs.find_all('time', {'class': 'ml-2 font-bold'})[0].text.strip().replace(",", "") + ":00"
  219.             return datetime.datetime.strftime(datetime.datetime.strptime(streamscharts_datetime, "%d %b %Y %H:%M:%S"), "%Y-%m-%d %H:%M:%S")
  220.  
  221.  
  222. def parse_datetime_twitchtracker(tracker_url):
  223.     response = requests.get(tracker_url, headers=return_header(), allow_redirects=False)
  224.     bs = BeautifulSoup(response.content, 'html.parser')
  225.     twitchtracker_datetime = bs.find_all('div', {'class': 'stream-timestamp-dt'})[0].text
  226.     return twitchtracker_datetime
  227.  
  228.  
  229. def parse_datetime_sullygnome(tracker_url):
  230.     response = requests.get(tracker_url, headers=return_header(), allow_redirects=False)
  231.     bs = BeautifulSoup(response.content, 'html.parser')
  232.     stream_date = bs.find_all('div', {'class': 'MiddleSubHeaderItemValue'})[6].text
  233.     if len(stream_date.split(" ")[1]) > 3:
  234.         day = stream_date.split(" ")[1][:2]
  235.     else:
  236.         day = stream_date.split(" ")[1][:1]
  237.     month = stream_date.split(" ")[2]
  238.     year = datetime.datetime.today().year
  239.     timestamp = stream_date.split(" ")[3]
  240.     stream_datetime = day + " " + month + " " + str(year) + " " + timestamp
  241.     sullygnome_datetime = datetime.datetime.strftime(datetime.datetime.strptime(stream_datetime, "%d %B %Y %I:%M%p"), "%Y-%m-%d %H:%M:%S")
  242.     return sullygnome_datetime
  243.  
  244.  
  245. def unmute_vod(url):
  246.     file_contents = []
  247.     counter = 0
  248.     vod_file_path = generate_vod_filename(return_username(url), return_vod_id(url))
  249.     with open(vod_file_path, "w") as vod_file:
  250.         vod_file.write(requests.get(url, stream=True).text)
  251.     vod_file.close()
  252.     with open(vod_file_path, "r") as vod_file:
  253.         for lines in vod_file.readlines():
  254.             file_contents.append(lines)
  255.     vod_file.close()
  256.     with open(vod_file_path, "w") as vod_file:
  257.         for segment in file_contents:
  258.             url = url.replace("index-dvr.m3u8", "")
  259.             if "-unmuted" in segment and not segment.startswith("#"):
  260.                 counter += 1
  261.                 vod_file.write(segment.replace(segment, str(url) + str(counter - 1)) + "-muted.ts" + "\n")
  262.             elif "-unmuted" not in segment and not segment.startswith("#"):
  263.                 counter += 1
  264.                 vod_file.write(segment.replace(segment, str(url) + str(counter - 1)) + ".ts" + "\n")
  265.             else:
  266.                 vod_file.write(segment)
  267.     vod_file.close()
  268.     print(os.path.basename(vod_file_path) + " Has been unmuted!")
  269.  
  270.  
  271. def dump_playlist(url):
  272.     file_contents = []
  273.     counter = 0
  274.     vod_file_path = generate_vod_filename(return_username(url), return_vod_id(url))
  275.     with open(vod_file_path, "w") as vod_file:
  276.         vod_file.write(requests.get(url, stream=True).text)
  277.     vod_file.close()
  278.     with open(vod_file_path, "r") as vod_file:
  279.         for lines in vod_file.readlines():
  280.             file_contents.append(lines)
  281.     vod_file.close()
  282.     with open(vod_file_path, "w") as vod_file:
  283.         for segment in file_contents:
  284.             url = url.replace("index-dvr.m3u8", "")
  285.             if not segment.startswith("#"):
  286.                 counter += 1
  287.                 vod_file.write(segment.replace(segment, str(url) + str(counter - 1)) + ".ts" + "\n")
  288.             else:
  289.                 vod_file.write(segment)
  290.     vod_file.close()
  291.  
  292.  
  293. def return_valid_file(url):
  294.     if is_vod_muted(url):
  295.         print("Vod contains muted segments")
  296.         unmute_vod(url)
  297.     else:
  298.         print("Vod does NOT contain muted segments")
  299.         dump_playlist(url)
  300.     new_playlist = []
  301.     vod_file_path = generate_vod_filename(return_username(url), return_vod_id(url))
  302.     new_vod_file_path = generate_vod_filename(return_username(url), return_vod_id(url) + "_MODIFIED")
  303.     lines = open(vod_file_path, "r+").read().splitlines()
  304.     segments = get_valid_segments(get_all_playlist_segments(url))
  305.     if len(segments) < 1:
  306.         print("No segments are valid.. Cannot generate M3U8! Returning to main menu.")
  307.         remove_file(vod_file_path)
  308.         return
  309.     new_playlist_segments = [x for x in segments if x in lines]
  310.     for segment in natsorted(new_playlist_segments):
  311.         for line in lines:
  312.             if line == segment:
  313.                 new_playlist.append(segment)
  314.             if line != segment and line.startswith("#"):
  315.                 new_playlist.append(line)
  316.             elif line.endswith(".ts") and segment not in new_playlist and not line.startswith("#"):
  317.                 line = "#" + line
  318.                 new_playlist.append(line)
  319.             else:
  320.                 if line not in new_playlist:
  321.                     new_playlist.append(line)
  322.         break
  323.     with open(new_vod_file_path, "a+") as new_vod_file:
  324.         for playlist_lines in new_playlist:
  325.             new_vod_file.write(playlist_lines + "\n")
  326.     new_vod_file.close()
  327.     remove_file(vod_file_path)
  328.  
  329.  
  330. def get_all_playlist_segments(url):
  331.     counter = 0
  332.     file_contents, segment_list = [], []
  333.     vod_file_path = generate_vod_filename(return_username(url), return_vod_id(url))
  334.     with open(vod_file_path, "w") as vod_file:
  335.         vod_file.write(requests.get(url, stream=True).text)
  336.     vod_file.close()
  337.     with open(vod_file_path, "r") as vod_file:
  338.         for lines in vod_file.readlines():
  339.             file_contents.append(lines)
  340.     vod_file.close()
  341.     with open(vod_file_path, "w") as vod_file:
  342.         for segment in file_contents:
  343.             url = url.replace("index-dvr.m3u8", "")
  344.             if "-unmuted" in segment and not segment.startswith("#"):
  345.                 counter += 1
  346.                 vod_file.write(segment.replace(segment, str(url) + str(counter - 1)) + "-muted.ts" + "\n")
  347.                 segment_list.append(str(url) + str(counter - 1) + "-muted.ts")
  348.             elif "-unmuted" not in segment and not segment.startswith("#"):
  349.                 counter += 1
  350.                 vod_file.write(segment.replace(segment, str(url) + str(counter - 1)) + ".ts" + "\n")
  351.                 segment_list.append(str(url) + str(counter - 1) + ".ts")
  352.             else:
  353.                 vod_file.write(segment)
  354.     vod_file.close()
  355.     return segment_list
  356.  
  357.  
  358. def get_valid_segments(segments):
  359.     valid_segment_counter, current_count = 0, 0
  360.     all_segments, valid_segments = [], []
  361.     for url in segments:
  362.         all_segments.append(url.strip())
  363.     request_session = requests.Session()
  364.     rs = [grequests.head(u, session=request_session) for u in all_segments]
  365.     for result in grequests.imap(rs, size=100):
  366.         current_count += 1
  367.         progress_percentage = (current_count * 100) // len(all_segments)
  368.         if current_count == len(all_segments):
  369.             print("\rChecking segment ", current_count, "/", len(all_segments), "... (progress : ", progress_percentage, "%)", sep='')
  370.         else:
  371.             print("\rChecking segment ", current_count, "/", len(all_segments), "... (progress : ", progress_percentage, "%)", sep='', end='')
  372.         if result.status_code == 200:
  373.             valid_segment_counter += 1
  374.             valid_segments.append(result.url)
  375.     return valid_segments
  376.  
  377.  
  378. def return_segment_ratio(url):
  379.     segment_string = str(len(get_valid_segments(get_all_playlist_segments(url)))) + " of " + str(
  380.         len(get_all_playlist_segments(url))) + " Segments are valid"
  381.     print(segment_string)
  382.  
  383.  
  384. def vod_recover(streamer, vod_id, timestamp):
  385.     print("Vod is " + str(get_vod_age(timestamp)) + " days old. If the vod is older than 60 days chances of recovery are slim." + "\n")
  386.     vod_url_list = get_vod_urls(streamer, vod_id, timestamp)
  387.     if len(vod_url_list) > 0:
  388.         vod_url = random.choice(vod_url_list)
  389.         if is_vod_muted(vod_url):
  390.             print(vod_url + "\n" + "Vod contains muted segments")
  391.             user_input = input("Would you like to unmute the vod (Y/N): ")
  392.             if user_input.upper() == "Y":
  393.                 unmute_vod(vod_url)
  394.                 print("Total Number of Segments: " + str(len(get_all_playlist_segments(vod_url))))
  395.                 user_option = input("Would you like to check if segments are valid (Y/N): ")
  396.                 if user_option.upper() == "Y":
  397.                     return_segment_ratio(vod_url)
  398.                 else:
  399.                     return
  400.             else:
  401.                 return
  402.         else:
  403.             print(vod_url + "\n" + "Vod does NOT contain muted segments")
  404.             print("Total Number of Segments: " + str(len(get_all_playlist_segments(vod_url))))
  405.             user_option = input("Would you like to check if segments are valid (Y/N): ")
  406.             if user_option.upper() == "Y":
  407.                 return_segment_ratio(vod_url)
  408.             else:
  409.                 return
  410.     else:
  411.         print(
  412.             "No vods found using current domain list. " + "\n" + "See the following links if you would like to check the other sites: " + "\n")
  413.         for website in generate_website_links(streamer, vod_id):
  414.             print(website)
  415.  
  416.  
  417. def manual_vod_recover():
  418.     streamer = input("Enter streamer name: ")
  419.     vod_id = input("Enter vod id: ")
  420.     timestamp = input("Enter VOD start time (YYYY-MM-DD HH:MM:SS): ")
  421.     vod_recover(streamer, vod_id, timestamp)
  422.  
  423.  
  424. def website_vod_recover():
  425.     tracker_url = input("Enter twitchtracker/streamscharts/sullygnome url:  ")
  426.     if "streamscharts" in tracker_url:
  427.         streamer = tracker_url.split("channels/", 1)[1].split("/")[0]
  428.         vod_id = tracker_url.split("streams/", 1)[1]
  429.         vod_recover(streamer, vod_id, parse_datetime_streamscharts(tracker_url))
  430.     elif "twitchtracker" in tracker_url:
  431.         streamer = tracker_url.split("com/", 1)[1].split("/")[0]
  432.         vod_id = tracker_url.split("streams/", 1)[1]
  433.         vod_recover(streamer, vod_id, parse_datetime_twitchtracker(tracker_url))
  434.     elif "sullygnome" in tracker_url:
  435.         streamer = tracker_url.split("channel/", 1)[1].split("/")[0]
  436.         vod_id = tracker_url.split("stream/", 1)[1]
  437.         vod_recover(streamer, vod_id, parse_datetime_sullygnome(tracker_url))
  438.     else:
  439.         print("Link not supported.. Returning to main menu.")
  440.         return
  441.  
  442.  
  443. def website_clip_recover():
  444.     tracker_url = input("Enter twitchtracker/streamscharts/sullygnome url:  ")
  445.     if "streamscharts" in tracker_url:
  446.         streamer = tracker_url.split("channels/", 1)[1].split("/")[0]
  447.         vod_id = tracker_url.split("streams/", 1)[1]
  448.         clip_recover(streamer, vod_id, int(parse_duration_streamscharts(tracker_url)))
  449.     elif "twitchtracker" in tracker_url:
  450.         streamer = tracker_url.split("com/", 1)[1].split("/")[0]
  451.         vod_id = tracker_url.split("streams/", 1)[1]
  452.         clip_recover(streamer, vod_id, int(parse_duration_twitchtracker(tracker_url)))
  453.     elif "sullygnome" in tracker_url:
  454.         streamer = tracker_url.split("channel/", 1)[1].split("/")[0]
  455.         vod_id = tracker_url.split("stream/", 1)[1]
  456.         clip_recover(streamer, vod_id, int(parse_duration_sullygnome(tracker_url)))
  457.     else:
  458.         print("Link not supported.. Returning to main menu.")
  459.         return
  460.  
  461.  
  462. def bulk_vod_recovery():
  463.     streamer = input("Enter streamer name: ")
  464.     file_path = input("Enter full path of sullygnome CSV file: ").replace('"', '')
  465.     for timestamp, vod_id in parse_vod_csv_file(file_path).items():
  466.         print("\n" + "Recovering Vod....", vod_id)
  467.         vod_url_list = get_vod_urls(streamer, vod_id, timestamp)
  468.         if len(vod_url_list) > 0:
  469.             vod_url = random.choice(vod_url_list)
  470.             if is_vod_muted(vod_url):
  471.                 print(vod_url + "\n" + "Vod contains muted segments")
  472.             else:
  473.                 print(vod_url + "\n" + "Vod does NOT contain muted segments")
  474.         else:
  475.             print("No vods found using current domain list." + "\n")
  476.  
  477.  
  478. def clip_recover(streamer, vod_id, duration):
  479.     total_counter, iteration_counter, valid_counter = 0, 0, 0
  480.     valid_url_list = []
  481.     clip_format = input("What clip url format would you like to use (delimited by spaces)? " + "\n" + "1) Default ([VodID]-offset-[interval])" + "\n" + "2) Alternate Format (vod-[VodID]-offset-[interval])" + "\n" + "3) Legacy ([VodID]-index-[interval])" + "\n").split()
  482.     full_url_list = get_all_clip_urls(get_clip_format(vod_id, get_reps(duration)), clip_format)
  483.     request_session = requests.Session()
  484.     rs = [grequests.head(u, session=request_session) for u in full_url_list]
  485.     for result in grequests.imap(rs, size=100):
  486.         total_counter += 1
  487.         iteration_counter += 1
  488.         if total_counter == 500:
  489.             print(str(iteration_counter) + " of " + str(len(full_url_list)))
  490.             total_counter = 0
  491.         if result.status_code == 200:
  492.             valid_counter += 1
  493.             valid_url_list.append(result.url)
  494.             print(str(valid_counter) + " Clip(s) Found")
  495.     if len(valid_url_list) >= 1:
  496.         with open(generate_log_filename(streamer, vod_id), "w") as log_file:
  497.             for url in valid_url_list:
  498.                 log_file.write(url + "\n")
  499.         log_file.close()
  500.         download_option = input("Do you want to download the recovered clips (Y/N): ")
  501.         if download_option.upper() == "Y":
  502.             download_clips(get_default_directory(), streamer, vod_id)
  503.             keep_log_option = input("Do you want to remove the log file? ")
  504.             if keep_log_option.upper() == "Y":
  505.                 remove_file(generate_log_filename(streamer, vod_id))
  506.             else:
  507.                 pass
  508.         else:
  509.             return
  510.     else:
  511.         print("No clips found! Returning to main menu.")
  512.         return
  513.  
  514.  
  515. def manual_clip_recover():
  516.     streamer = input("Enter streamer name: ")
  517.     vod_id = input("Enter vod id: ")
  518.     hours = input("Enter stream duration hour value: ")
  519.     minutes = input("Enter stream duration minute value: ")
  520.     clip_recover(streamer, vod_id, get_duration(hours, minutes))
  521.  
  522.  
  523. def parse_clip_csv_file(file_path):
  524.     vod_info_dict = {}
  525.     csv_file = open(file_path, "r+")
  526.     lines = csv_file.readlines()[1:]
  527.     for line in lines:
  528.         if line.strip():
  529.             filtered_string = line.partition("stream/")[2].replace('"', "")
  530.             vod_id = filtered_string.split(",")[0]
  531.             duration = filtered_string.split(",")[1]
  532.             if vod_id != 0:
  533.                 reps = get_reps(int(duration))
  534.                 vod_info_dict.update({vod_id: reps})
  535.             else:
  536.                 pass
  537.     csv_file.close()
  538.     return vod_info_dict
  539.  
  540.  
  541. def parse_vod_csv_file(file_path):
  542.     vod_info_dict = {}
  543.     csv_file = open(file_path, "r+")
  544.     lines = csv_file.readlines()[1:]
  545.     for line in lines:
  546.         if line.strip():
  547.             if len(line.split(",")[1].split(" ")[1]) > 3:
  548.                 day = line.split(",")[1].split(" ")[1][:2]
  549.             else:
  550.                 day = line.split(",")[1].split(" ")[1][:1]
  551.             month = line.split(",")[1].split(" ")[2]
  552.             year = line.split(",")[1].split(" ")[3]
  553.             timestamp = line.split(",")[1].split(" ")[4]
  554.             stream_datetime = day + " " + month + " " + year + " " + timestamp
  555.             vod_id = line.partition("stream/")[2].split(",")[0].replace('"', "")
  556.             stream_date = datetime.datetime.strftime(
  557.                 datetime.datetime.strptime(stream_datetime.strip().replace('"', "") + ":00", "%d %B %Y %H:%M:%S"),
  558.                 "%Y-%m-%d %H:%M:%S")
  559.             vod_info_dict.update({stream_date: vod_id})
  560.     csv_file.close()
  561.     return vod_info_dict
  562.  
  563.  
  564. def get_random_clips():
  565.     counter = 0
  566.     vod_id = input("Enter vod id: ")
  567.     hours = input("Enter stream duration hour value: ")
  568.     minutes = input("Enter stream duration minute value: ")
  569.     clip_format = input("What clip url format would you like to use (delimited by spaces)? " + "\n" + "1) Default ([VodID]-offset-[interval])" + "\n" + "2) Alternate Format (vod-[VodID]-offset-[interval])" + "\n" + "3) Legacy ([VodID]-index-[interval])" + "\n").split()
  570.     full_url_list = get_all_clip_urls(get_clip_format(vod_id, get_reps(get_duration(hours, minutes))), clip_format)
  571.     random.shuffle(full_url_list)
  572.     print("Total Number of Urls: " + str(len(full_url_list)))
  573.     request_session = requests.Session()
  574.     rs = [grequests.head(u, session=request_session) for u in full_url_list]
  575.     for result in grequests.imap(rs, size=100):
  576.         if result.status_code == 200:
  577.             counter += 1
  578.             if counter == 1:
  579.                 print(result.url)
  580.                 user_option = input("Do you want another url (Y/N): ")
  581.                 if user_option.upper() == "Y":
  582.                     continue
  583.                 else:
  584.                     return
  585.         counter = 0
  586.  
  587.  
  588. def bulk_clip_recovery():
  589.     vod_counter, total_counter, valid_counter, iteration_counter = 0, 0, 0, 0
  590.     streamer = input("Enter streamer name: ")
  591.     file_path = input("Enter full path of sullygnome CSV file: ").replace('"', '')
  592.     user_option = input("Do you want to download all clips recovered (Y/N)? ")
  593.     clip_format = input("What clip url format would you like to use (delimited by spaces)? " + "\n" + "1) Default ([VodID]-offset-[interval])" + "\n" + "2) Alternate Format (vod-[VodID]-offset-[interval])" + "\n" + "3) Legacy ([VodID]-index-[interval])" + "\n").split()
  594.     for vod_id, duration in parse_clip_csv_file(file_path).items():
  595.         vod_counter += 1
  596.         print("Processing Twitch Vod... " + str(vod_id) + " - " + str(vod_counter) + " of " + str(
  597.             len(parse_clip_csv_file(file_path))))
  598.         original_vod_url_list = get_all_clip_urls(get_clip_format(vod_id, duration), clip_format)
  599.         request_session = requests.Session()
  600.         rs = [grequests.head(u, session=request_session) for u in original_vod_url_list]
  601.         for result in grequests.imap(rs, size=100):
  602.             total_counter += 1
  603.             iteration_counter += 1
  604.             if total_counter == 500:
  605.                 print(str(iteration_counter) + " of " + str(len(original_vod_url_list)))
  606.                 total_counter = 0
  607.             if result.status_code == 200:
  608.                 valid_counter += 1
  609.                 print(str(valid_counter) + " Clip(s) Found")
  610.                 with open(generate_log_filename(streamer, vod_id), "a+") as log_file:
  611.                     log_file.write(result.url + "\n")
  612.                 log_file.close()
  613.             else:
  614.                 continue
  615.         if valid_counter != 0:
  616.             if user_option.upper() == "Y":
  617.                 download_clips(get_default_directory(), streamer, vod_id)
  618.                 remove_file(generate_log_filename(streamer, vod_id))
  619.             else:
  620.                 print("Recovered clips logged to " + generate_log_filename(streamer, vod_id))
  621.         total_counter, valid_counter, iteration_counter = 0, 0, 0
  622.  
  623.  
  624. def download_m3u8(url):
  625.     videos = []
  626.     ts_video_list = natsorted(get_valid_segments(get_all_playlist_segments(url)))
  627.     for ts_files in ts_video_list:
  628.         print("Processing.... " + ts_files)
  629.         if ts_files.endswith(".ts"):
  630.             video = VideoFileClip(ts_files)
  631.             videos.append(video)
  632.     final_vod_output = concatenate_videoclips(videos)
  633.     final_vod_output.to_videofile(os.path.join(get_default_directory(), "VodRecovery_" + return_username(url) + "_" + return_vod_id(url) + ".mp4"), fps=60, remove_temp=True)
  634.  
  635.  
  636. def download_clips(directory, streamer, vod_id):
  637.     counter = 0
  638.     print("Starting Download....")
  639.     download_directory = os.path.join(directory, streamer.title() + "_" + vod_id)
  640.     if os.path.exists(download_directory):
  641.         pass
  642.     else:
  643.         os.mkdir(download_directory)
  644.     for links in return_file_contents(streamer, vod_id):
  645.         counter = counter + 1
  646.         if "-offset-" in links:
  647.             clip_offset = links.split("-offset-")[1].replace(".mp4", "")
  648.         else:
  649.             clip_offset = links.split("-index-")[1].replace(".mp4", "")
  650.         link_url = os.path.basename(links)
  651.         r = requests.get(links, stream=True)
  652.         if r.status_code == 200:
  653.             if str(link_url).endswith(".mp4"):
  654.                 with open(os.path.join(download_directory, streamer.title() + "_" + str(vod_id) + "_" + str(
  655.                         clip_offset)) + ".mp4", 'wb') as x:
  656.                     print(datetime.datetime.now().strftime("%Y/%m/%d %I:%M:%S    ") + "Downloading... Clip " + str(
  657.                         counter) + " of " + str(len(return_file_contents(streamer, vod_id))) + " - " + links)
  658.                     x.write(r.content)
  659.             else:
  660.                 print("ERROR: Please check the log file and failing link!", links)
  661.         else:
  662.             print("ERROR: " + str(r.status_code) + " - " + str(r.reason))
  663.             pass
  664.  
  665.  
  666. def run_script():
  667.     menu = 0
  668.     while menu < 7:
  669.         return_main_menu()
  670.         menu = int(input("Please choose an option: "))
  671.         if menu == 7:
  672.             exit()
  673.         elif menu == 1:
  674.             vod_type = int(input(
  675.                 "Enter what type of vod recovery: " + "\n" + "1) Recover Vod" + "\n" + "2) Recover vods from SullyGnome CSV export" + "\n"))
  676.             if vod_type == 1:
  677.                 vod_recovery_method = int(input("Enter vod recovery method: " + "\n" + "1) Manual Vod Recover" + "\n" + "2) Website Vod Recover" + "\n"))
  678.                 if vod_recovery_method == 1:
  679.                     manual_vod_recover()
  680.                 elif vod_recovery_method == 2:
  681.                     website_vod_recover()
  682.                 else:
  683.                     print("Invalid option returning to main menu.")
  684.             elif vod_type == 2:
  685.                 bulk_vod_recovery()
  686.             else:
  687.                 print("Invalid option! Returning to main menu.")
  688.         elif menu == 2:
  689.             clip_type = int(input(
  690.                 "Enter what type of clip recovery: " + "\n" + "1) Recover all clips from a single VOD" + "\n" + "2) Find random clips from a single VOD" + "\n" + "3) Bulk recover clips from SullyGnome CSV export" + "\n"))
  691.             if clip_type == 1:
  692.                 clip_recovery_method = int(input("Enter clip recovery method: " + "\n" + "1) Manual Clip Recover" + "\n" + "2) Website Clip Recover" + "\n"))
  693.                 if clip_recovery_method == 1:
  694.                     manual_clip_recover()
  695.                 elif clip_recovery_method == 2:
  696.                     website_clip_recover()
  697.                 else:
  698.                     print("Invalid option returning to main menu.")
  699.             elif clip_type == 2:
  700.                 get_random_clips()
  701.             elif clip_type == 3:
  702.                 bulk_clip_recovery()
  703.             else:
  704.                 print("Invalid option! Returning to main menu.")
  705.         elif menu == 3:
  706.             url = input("Enter M3U8 Link: ")
  707.             if is_vod_muted(url):
  708.                 unmute_vod(url)
  709.             else:
  710.                 print("Vod does NOT contain muted segments")
  711.         elif menu == 4:
  712.             url = input("Enter M3U8 Link: ")
  713.             return_segment_ratio(url)
  714.             remove_file(generate_vod_filename(return_username(url), return_vod_id(url)))
  715.         elif menu == 5:
  716.             url = input("Enter M3U8 Link: ")
  717.             return_valid_file(url)
  718.         elif menu == 6:
  719.             url = input("Enter M3U8 Link: ")
  720.             download_m3u8(url)
  721.         else:
  722.             print("Invalid Option! Exiting...")
  723.  
  724.  
  725. run_script()
  726.  
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!