DaEvil1

Untitled

Sep 11th, 2014
249
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 6.33 KB | None | 0 0
  1. import urllib.request, praw, csv, time, sys, datetime, string
  2.  
  3. from bs4 import BeautifulSoup
  4.  
  5. title_s = "title:"
  6. description_s = "description:"
  7. type_s = "type:"
  8. map_s = "map:"
  9. preview_s = "preview:"
  10.  
  11.  
  12. def get_comments(thread_id):
  13.     r = praw.Reddit("Maptest Committee Extractor")
  14.     thread_data = r.get_submission(submission_id = thread_id)
  15.     thread_data.replace_more_comments(limit = None, threshold = 0)
  16.     return thread_data.comments
  17.  
  18.  
  19. def get_string(comment_body, element):
  20.     while "\n " in comment_body:
  21.         comment_body = comment_body.replace("\n ", "\n")
  22.     comment_body = comment_body.replace("  \n", "\n\n")
  23.     comment_list = comment_body.split("\n\n")
  24.     for i in range(len(comment_list)):
  25.         comment_list[i] = comment_list[i].replace("\n", "")
  26.         strip_comment = comment_list[i].replace("*", "")
  27.         l_comment = strip_comment.lower()
  28.         current_pos = l_comment.find(element)
  29.         if current_pos != -1:
  30.             modi = 0
  31.             http_pos = comment_list[i].find("http")
  32.             end_website = comment_list[i][http_pos:].find(" ")
  33.             if http_pos != -1:
  34.                 beg_com = comment_list[i][:http_pos].replace("*", "")
  35.                 end_com = comment_list[i][end_website:].replace("*", "")
  36.                 comment_list[i] = beg_com + \
  37.                                   comment_list[i][http_pos:end_website] + \
  38.                                   end_com
  39.             else:
  40.                 comment_list[i] = comment_list[i].replace("*", "")
  41.             while ": " in comment_list[i]:
  42.                 comment_list[i] = comment_list[i].replace(": ", ":")
  43.             current_pos += len(element)
  44.             return comment_list[i][current_pos:]
  45.     return False
  46.  
  47.  
  48. def get_comment_data(comment):
  49.     comment_data = {}
  50.     comment_info = [title_s, description_s, type_s, map_s, preview_s]
  51.     for i in comment_info:
  52.         new_data = get_string(comment.body, i)
  53.         comment_data[i[:-1]] = new_data
  54.     comment_data["timestamp"] = comment.created_utc
  55.     comment_data["link"] = comment.permalink
  56.     return comment_data
  57.  
  58. def fix_bad_info(maps):
  59.     bad_titles = 0
  60.     map_info = [description_s, type_s, map_s, preview_s]
  61.     base_s = "Missing "
  62.     for i in maps:
  63.         if not i["title"]:
  64.             bad_titles += 1
  65.             new_title = "Missing title " + str(bad_titles)
  66.             i["title"] = new_title
  67.         if i["preview"]:
  68.             if "(" in i["preview"] and ")" in i["preview"]:
  69.                 start_pos = i["preview"].find("(")
  70.                 end_pos = i["preview"].find(")")
  71.                 i["preview"] = i["preview"][start_pos + 1:end_pos]
  72.         if i["map"]:
  73.             if "(" in i["map"] and ")" in i["map"]:
  74.                 start_pos = i["map"].find("(")
  75.                 end_pos = i["map"].find(")")
  76.                 i["map"] = i["map"][start_pos + 1:end_pos]
  77.     return maps
  78.  
  79.  
  80. def sort_maps(maps):
  81.     maps = [i for i in maps]
  82.     new_maps = []
  83.     sorted_maps = []
  84.     while len(maps) > 0:
  85.         current_lowest = maps[0]
  86.         lowest = maps[0]["timestamp"]*1000000000
  87.         current_map = None
  88.         for element in maps:
  89.             if element["title"] not in sorted_maps:
  90.                 if element["timestamp"] <= lowest:
  91.                     current_lowest = element
  92.                     lowest = element["timestamp"]
  93.         sorted_maps.append(current_lowest["title"])
  94.         new_maps.append(current_lowest)
  95.         maps.remove(current_lowest)
  96.     return new_maps
  97.  
  98. def get_map_id(url):
  99.     try:
  100.         data = urllib.request.urlopen(url)
  101.         page = BeautifulSoup(data)
  102.         return page.find("a", "testmap")["mapid"]
  103.     except:
  104.         return False
  105.  
  106.  
  107.  
  108. def fix_image(image):
  109.     if "imgur" in image and not "i.imgur" in image:
  110.         image = image[:7] + "i." + image[7:] + ".png"
  111.         image = image.replace(" ", "")
  112.     return image
  113.  
  114.  
  115. def write_csv(maps):
  116.     out_strings = []
  117.     n_maps = 0
  118.     test_str1 =  "=HYPERLINK(\"http://tagpromaptest.appspot.com/somebot/"
  119.     test_str2 = "\", \"Test Map\")"
  120.     image_str1 = "=image(\""
  121.     image_str2 = "\", 1)"
  122.     preview_str1 = "=HYPERLINK(\""
  123.     preview_str2 = "\", \"Preview\")"
  124.     link_str1 = "=HYPERLINK(\""
  125.     link_str2 = "\", \""
  126.     link_str3 = "\")"
  127.     i = 0
  128.     for cur_map in maps:
  129.         n_maps += 1
  130.         map_id = get_map_id(cur_map["map"])
  131.         test_map = "Improper Test Format"
  132.         if map_id:
  133.             test_map = test_str1 + map_id + test_str2
  134.         else:
  135.             if cur_map["map"]:
  136.                 test_map = cur_map["map"]
  137.         map_image = "Improper Image Link"
  138.         map_preview = "improper Image Link"
  139.         if cur_map["preview"]:
  140.             cur_map["preview"] = fix_image(cur_map["preview"])
  141.             map_image = image_str1 + cur_map["preview"] + image_str2
  142.             map_preview = preview_str1 + cur_map["preview"] + preview_str2
  143.         map_desc = "No Description"
  144.         if cur_map["description"]:
  145.             map_desc = cur_map["description"]
  146.         map_link = link_str1 + cur_map["link"] + link_str2
  147.         map_link += cur_map["title"] + link_str3
  148.         map_type = "No Type Given"
  149.         if cur_map["type"]:
  150.             map_type = cur_map["type"]
  151.         out = [map_link, map_type, test_map, map_image, map_preview, map_desc]
  152.         out_strings.append(out)
  153.     filename = "maps_{}.txt".format(datetime.date.today().strftime("%B_%d_%Y"))
  154.     with open(filename, "w") as f:
  155.         a = csv.writer(f, delimiter='\t', lineterminator='\n')
  156.         a.writerows(out_strings)
  157.     return filename, n_maps
  158.  
  159. def summary(filename, n_maps, maps):
  160.     print("Filename:", filename, "maps:", n_maps, "Posts:", len(maps))
  161.  
  162.  
  163. def main():
  164.     thread_id = sys.argv[1]
  165.     sys.stdout.write("Getting comments... ")
  166.     comments = get_comments(thread_id)
  167.     sys.stdout.write("Done\nParsing comment data...")
  168.     maps = []
  169.     for comment in comments:
  170.         comment_data = get_comment_data(comment)
  171.         maps.append(comment_data)
  172.     sys.stdout.write("Done\nFixing Bad Info...")
  173.     maps = fix_bad_info(maps)
  174.     sys.stdout.write("Done\nSorting Maps...")
  175.     maps_sorted = sort_maps(maps)
  176.     sys.stdout.write("Done\nWriting csv...\n")
  177.     filename, n_maps = write_csv(maps_sorted)
  178.     sys.stdout.write("Done\n")
  179.     summary(filename, n_maps, maps)
  180.  
  181.  
  182. if __name__ == "__main__":
  183.     main()
Advertisement
Add Comment
Please, Sign In to add comment