get-subreddit.py

import praw

import argparse

import json

class GetSubreddit():
    user_agent = "gnu/linux:get-subreddit:v1.0 (by /u/A_Truth_Value)"
    def main(self):
        parser = argparse.ArgumentParser()
        parser.add_argument("subreddit", help="The subreddit to grab.")
        parser.add_argument("-o", "--output", help="Directory to write JSON "
                            "to.")
        arguments = parser.parse_args()
        if arguments.subreddit:
            rconnection = praw.Reddit(self.user_agent)
            submissions = rconnection.get_subreddit(arguments.subreddit).get_hot(
                limit=None)
            posts = []
            for value in submissions:
                posts.append(value)
            description = ("The JSON data format I've chosen is slim but"
                           "represents the essential attributes of posts"
                           "and submissions. For submissions it is as follows:"
                           "\n\n"
                           "id: The unique id of the submission."
                           "title: The title of the submission.\n"
                           "body: For text posts, the body of the submission."
                           "\n"
                           "time_created: The time of creation.\n"
                           "submitted_by: The username of the submitter.\n"
                           "score: The score of the submission.\n"
                           "url: The content-link of the submission.\n"
                           "permalink: The original reddit link of the "
                           "submission\n"
                           "root_replies: A dictionary containing the top "
                           "level replies for this submission. The keys of the"
                           " dictionary are the unique id of each post.\n"
                           "For comments a *tree* is constructed starting with"
                           " a root post and then the child posts of that and"
                           " their child posts recursively until a complete"
                           " comment tree is constructed. Attributes are:\n\n"
                           "id: The unique id of the comment."
                           "body: The body of the comment."
                           "author: The user thet wrote the comment."
                           "time_posted: The time that the comment was posted."
                           "score: The comments score."
                           "replies: A dictionary of replies. Null if none.")
            submission_dicts = {'description':description}
            for submission in posts:
                submission_dict = {}
                submission_dict["title"] = submission.title
                submission_dict["body"] = submission.title
                submission_dict["time_created"] = submission.created_utc
                submission_dict["submitted_by"] = submission.author.name
                submission_dict["score"] = submission.score
                submission_dict["url"] = submission.url
                submission_dict["permalink"] = submission.permalink
                submission_dict["root_replies"] = self.create_tree(submission.
                                                                   comments)
                submission_dicts[submission.id] = submission_dict
            outfile = open(arguments.output, 'w')
            json.dump(submission_dicts, outfile)
        else:
            raise ValueError("You didn't specify a subreddit to grab.")

    def create_tree(self, replies):
        tree_nodes = {}
        for reply in replies:
            if isinstance(reply, praw.objects.MoreComments):
                comments = reply.comments()
                reply = comments[0]
                replies.append(comments[1:])
            elif isinstance(reply, list):
                if reply:
                    raise ValueError("THE LIST IS REAL!")
                else:
                    print("Turns out it was junk.")
                    return None
            tree_node = {}
            try:
                tree_node["body"] = reply.body
            except:
                raise ValueError("Assumption made about data was wrong. O_o")
            if reply.author:
                tree_node["author"] = reply.author.name
            else:
                tree_node["author"] = reply.author
            tree_node["created_utc"] = reply.created_utc
            tree_node["score"] = reply.score
            if reply.replies:
                tree_node["replies"] = self.create_tree(reply.replies)
            else:
                tree_node["replies"] = None
            tree_nodes[reply.id] = tree_node
        if not replies:
            return None
        print(tree_nodes)
        return tree_nodes

getsub = GetSubreddit()
getsub.main()