Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # A Reddit bot that posts explanation of xkcd comic strips posted in comments
- # The explanation is extracted from http://explainxkcd.com
- # Created by Ayush Dwivedi (/u/kindw)
- # License: MIT License
- from bs4 import BeautifulSoup
- from urllib.parse import urlparse
- import praw
- import time
- import re
- import requests
- import bs4
- path = './commented.txt'
- # Location of file where id's of already visited comments are maintained
- header = '**Explanation of this xkcd:**\n'
- footer = '\n*---This explanation was extracted from [explainxkcd](http://www.explainxkcd.com) | Bot created by u/kindw | [Source code](https://github.com/aydwi/explainxkcdbot)*'
- # Text to be posted along with comic description
- def authenticate():
- print('Authenticating...\n')
- reddit = praw.Reddit('scrippy', user_agent = 'web:xkcd-explain-bot:v0.1 (by /u/kindw)')
- print('Authenticated as {}\n'.format(reddit.user.me()))
- return reddit
- #authenticate()
- reddit = authenticate()
- #------------scraping section------------
- def fetchData(url):
- r = requests.get(url)
- #there needs to be some error handling here with r.ok to handle timeouts and 404s or just the site being down
- print('Status code: {}\n'.format(r.status_code))
- #passing content of the request to BeautifulSoup
- soup = BeautifulSoup(r.content,'html.parser')
- tag = soup.find('p')# tag to start at
- data = ''# aggregrated data
- while True:
- if isinstance(tag,bs4.element.Tag):
- if(tag.name == 'h2'):# tag to end at
- break
- if(tag.name == 'h3'):# tag to skip
- tag = tag.next_sibling
- else:
- data = data + '\n' + tag.text
- tag = tag.next_sibling
- else:
- tag = tag.next_sibling
- return data
- #this should scrape all the text on the page, starting at first <p> up to the first <h2> tag
- testData = fetchData('http://explainxkcd.com/1024')
- print(testData)
- #I'll be passing my reddit object into here
- def run_scrippy(reddit):
- print("Getting 100 comments...\n")
- for comment in reddit.subreddit('test').comments(limit = 100):
- print(comment)
- match = re.findall("[a-z]*[A-Z]*[0-9]*https://www.xkcd.com/[0-9]+", comment.body)
- if match:
- print("Link found in comment with comment ID: "+ comment.id)
- xkcd_url = match[0]
- url_obj = urlparse(xkcd_url)
- xkcd_id = int((url_obj.path.strip("/")))
- myurl = 'http://www.explainxkcd.com/wiki/index.php/' + str(xkcd_id)
- file_obj_r = open(path,'r')
- try:
- explanation = fetchData(myurl) # this would be the try to get the scraped data of a certain webpage or response
- except:
- print('Exception!!! possibly incorrect xkcd URL...\n')
- # Typical cause for this will be a URL for an xkcd that does not exist (Example: https://www.xkcd.com/772524318/)
- else:
- if comment.id not in file_obj_r.read().splitlines():
- print('Link is unique... posting explanation\n')
- comment.reply(header +explanation + footer)
- file_obj_r.close()
- file_obj_w = open(path,'a+')
- file_obj_w.write(comment.id + '\n')
- file_obj_w.close()
- else:
- print('Already visted link... no reply needed\n')
- time.sleep(10)
- print('Waiting 60 seconds...\n')
- time.sleep(60)
- def main():
- reddit = authenticate()
- while True:
- run_scrippy(reddit)
- if __name__ == '__main__':
- main()
Advertisement
Add Comment
Please, Sign In to add comment