Guest User

Untitled

a guest
Aug 16th, 2017
98
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 3.75 KB | None | 0 0
  1. # A Reddit bot that posts explanation of xkcd comic strips posted in comments
  2. # The explanation is extracted from http://explainxkcd.com
  3. # Created by Ayush Dwivedi (/u/kindw)
  4. # License: MIT License
  5.  
  6. from bs4 import BeautifulSoup
  7. from urllib.parse import urlparse
  8.  
  9. import praw
  10. import time
  11. import re
  12. import requests
  13. import bs4
  14.  
  15. path = './commented.txt'
  16. # Location of file where id's of already visited comments are maintained
  17.  
  18. header = '**Explanation of this xkcd:**\n'
  19. footer = '\n*---This explanation was extracted from [explainxkcd](http://www.explainxkcd.com) | Bot created by u/kindw | [Source code](https://github.com/aydwi/explainxkcdbot)*'
  20. # Text to be posted along with comic description
  21.  
  22. def authenticate():
  23.    
  24.     print('Authenticating...\n')
  25.     reddit = praw.Reddit('scrippy', user_agent = 'web:xkcd-explain-bot:v0.1 (by /u/kindw)')
  26.     print('Authenticated as {}\n'.format(reddit.user.me()))
  27.     return reddit
  28.  
  29. #authenticate()
  30. reddit = authenticate()
  31.  
  32. #------------scraping section------------
  33. def fetchData(url):
  34.     r = requests.get(url)
  35.     #there needs to be some error handling here with r.ok to handle timeouts and 404s or just the site being down
  36.     print('Status code: {}\n'.format(r.status_code))
  37.  
  38. #passing content of the request to BeautifulSoup
  39.     soup = BeautifulSoup(r.content,'html.parser')
  40.  
  41.     tag = soup.find('p')#                   tag to start at
  42.     data = ''#                              aggregrated data
  43.     while True:
  44.         if isinstance(tag,bs4.element.Tag):
  45.             if(tag.name == 'h2'):#          tag to end at
  46.                 break
  47.             if(tag.name == 'h3'):#          tag to skip
  48.                 tag = tag.next_sibling
  49.             else:
  50.                 data = data + '\n' + tag.text
  51.                 tag = tag.next_sibling
  52.         else:
  53.             tag = tag.next_sibling
  54.  
  55.     return data
  56.  
  57. #this should scrape all the text on the page, starting at first <p> up to the first <h2> tag
  58. testData = fetchData('http://explainxkcd.com/1024')
  59. print(testData)
  60.  
  61. #I'll be passing my reddit object into here
  62. def run_scrippy(reddit):
  63.    
  64.     print("Getting 100 comments...\n")
  65.  
  66.     for comment in reddit.subreddit('test').comments(limit = 100):
  67.         print(comment)
  68.         match = re.findall("[a-z]*[A-Z]*[0-9]*https://www.xkcd.com/[0-9]+", comment.body)
  69.         if match:
  70.             print("Link found in comment with comment ID: "+ comment.id)
  71.             xkcd_url = match[0]
  72.             url_obj = urlparse(xkcd_url)
  73.             xkcd_id = int((url_obj.path.strip("/")))
  74.             myurl = 'http://www.explainxkcd.com/wiki/index.php/' + str(xkcd_id)
  75.  
  76.             file_obj_r = open(path,'r')
  77.  
  78.             try:
  79.                 explanation = fetchData(myurl) # this would be the try to get the scraped data of a certain webpage or response
  80.             except:
  81.                 print('Exception!!! possibly incorrect xkcd URL...\n')
  82.                 # Typical cause for this will be a URL for an xkcd that does not exist (Example: https://www.xkcd.com/772524318/)
  83.             else:
  84.                 if comment.id not in file_obj_r.read().splitlines():
  85.                     print('Link is unique... posting explanation\n')
  86.                     comment.reply(header +explanation + footer)
  87.  
  88.                     file_obj_r.close()
  89.  
  90.                     file_obj_w = open(path,'a+')
  91.                     file_obj_w.write(comment.id + '\n')
  92.                     file_obj_w.close()
  93.  
  94.                 else:
  95.                     print('Already visted link... no reply needed\n')
  96.            
  97.             time.sleep(10)
  98.    
  99.     print('Waiting 60 seconds...\n')
  100.     time.sleep(60)
  101.  
  102. def main():
  103.     reddit = authenticate()
  104.     while True:
  105.         run_scrippy(reddit)
  106.  
  107. if __name__ == '__main__':
  108.     main()
Advertisement
Add Comment
Please, Sign In to add comment