Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import urllib.request, praw, csv, time, sys, datetime, string
- from bs4 import BeautifulSoup
- title_s = "title:"
- description_s = "description:"
- type_s = "type:"
- map_s = "map:"
- preview_s = "preview:"
- def get_comments(thread_id):
- r = praw.Reddit("Maptest Committee Extractor")
- thread_data = r.get_submission(submission_id = thread_id)
- thread_data.replace_more_comments(limit = None, threshold = 0)
- return thread_data.comments
- def get_string(comment_body, element):
- while "\n " in comment_body:
- comment_body = comment_body.replace("\n ", "\n")
- comment_body = comment_body.replace(" \n", "\n\n")
- comment_list = comment_body.split("\n\n")
- for i in range(len(comment_list)):
- comment_list[i] = comment_list[i].replace("\n", "")
- strip_comment = comment_list[i].replace("*", "")
- l_comment = strip_comment.lower()
- current_pos = l_comment.find(element)
- if current_pos != -1:
- modi = 0
- http_pos = comment_list[i].find("http")
- end_website = comment_list[i][http_pos:].find(" ")
- if http_pos != -1:
- beg_com = comment_list[i][:http_pos].replace("*", "")
- end_com = comment_list[i][end_website:].replace("*", "")
- comment_list[i] = beg_com + \
- comment_list[i][http_pos:end_website] + \
- end_com
- else:
- comment_list[i] = comment_list[i].replace("*", "")
- while ": " in comment_list[i]:
- comment_list[i] = comment_list[i].replace(": ", ":")
- current_pos += len(element)
- return comment_list[i][current_pos:]
- return False
- def get_comment_data(comment):
- comment_data = {}
- comment_info = [title_s, description_s, type_s, map_s, preview_s]
- for i in comment_info:
- new_data = get_string(comment.body, i)
- comment_data[i[:-1]] = new_data
- comment_data["timestamp"] = comment.created_utc
- comment_data["link"] = comment.permalink
- return comment_data
- def fix_bad_info(maps):
- bad_titles = 0
- map_info = [description_s, type_s, map_s, preview_s]
- base_s = "Missing "
- for i in maps:
- if not i["title"]:
- bad_titles += 1
- new_title = "Missing title " + str(bad_titles)
- i["title"] = new_title
- if i["preview"]:
- if "(" in i["preview"] and ")" in i["preview"]:
- start_pos = i["preview"].find("(")
- end_pos = i["preview"].find(")")
- i["preview"] = i["preview"][start_pos + 1:end_pos]
- if i["map"]:
- if "(" in i["map"] and ")" in i["map"]:
- start_pos = i["map"].find("(")
- end_pos = i["map"].find(")")
- i["map"] = i["map"][start_pos + 1:end_pos]
- return maps
- def sort_maps(maps):
- maps = [i for i in maps]
- new_maps = []
- sorted_maps = []
- while len(maps) > 0:
- current_lowest = maps[0]
- lowest = maps[0]["timestamp"]*1000000000
- current_map = None
- for element in maps:
- if element["title"] not in sorted_maps:
- if element["timestamp"] <= lowest:
- current_lowest = element
- lowest = element["timestamp"]
- sorted_maps.append(current_lowest["title"])
- new_maps.append(current_lowest)
- maps.remove(current_lowest)
- return new_maps
- def get_map_id(url):
- try:
- data = urllib.request.urlopen(url)
- page = BeautifulSoup(data)
- return page.find("a", "testmap")["mapid"]
- except:
- return False
- def fix_image(image):
- if "imgur" in image and not "i.imgur" in image:
- image = image[:7] + "i." + image[7:] + ".png"
- image = image.replace(" ", "")
- return image
- def write_csv(maps):
- out_strings = []
- n_maps = 0
- test_str1 = "=HYPERLINK(\"http://tagpromaptest.appspot.com/somebot/"
- test_str2 = "\", \"Test Map\")"
- image_str1 = "=image(\""
- image_str2 = "\", 1)"
- preview_str1 = "=HYPERLINK(\""
- preview_str2 = "\", \"Preview\")"
- link_str1 = "=HYPERLINK(\""
- link_str2 = "\", \""
- link_str3 = "\")"
- i = 0
- for cur_map in maps:
- n_maps += 1
- map_id = get_map_id(cur_map["map"])
- test_map = "Improper Test Format"
- if map_id:
- test_map = test_str1 + map_id + test_str2
- else:
- if cur_map["map"]:
- test_map = cur_map["map"]
- map_image = "Improper Image Link"
- map_preview = "improper Image Link"
- if cur_map["preview"]:
- cur_map["preview"] = fix_image(cur_map["preview"])
- map_image = image_str1 + cur_map["preview"] + image_str2
- map_preview = preview_str1 + cur_map["preview"] + preview_str2
- map_desc = "No Description"
- if cur_map["description"]:
- map_desc = cur_map["description"]
- map_link = link_str1 + cur_map["link"] + link_str2
- map_link += cur_map["title"] + link_str3
- map_type = "No Type Given"
- if cur_map["type"]:
- map_type = cur_map["type"]
- out = [map_link, map_type, test_map, map_image, map_preview, map_desc]
- out_strings.append(out)
- filename = "maps_{}.txt".format(datetime.date.today().strftime("%B_%d_%Y"))
- with open(filename, "w") as f:
- a = csv.writer(f, delimiter='\t', lineterminator='\n')
- a.writerows(out_strings)
- return filename, n_maps
- def summary(filename, n_maps, maps):
- print("Filename:", filename, "maps:", n_maps, "Posts:", len(maps))
- def main():
- thread_id = sys.argv[1]
- sys.stdout.write("Getting comments... ")
- comments = get_comments(thread_id)
- sys.stdout.write("Done\nParsing comment data...")
- maps = []
- for comment in comments:
- comment_data = get_comment_data(comment)
- maps.append(comment_data)
- sys.stdout.write("Done\nFixing Bad Info...")
- maps = fix_bad_info(maps)
- sys.stdout.write("Done\nSorting Maps...")
- maps_sorted = sort_maps(maps)
- sys.stdout.write("Done\nWriting csv...\n")
- filename, n_maps = write_csv(maps_sorted)
- sys.stdout.write("Done\n")
- summary(filename, n_maps, maps)
- if __name__ == "__main__":
- main()
Advertisement
Add Comment
Please, Sign In to add comment