RedditData.java

package reddit;

import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonToken;

public class RedditData {

    public static final long MIN_REQUEST_DELAY = 5050l;
    public static final String USER_HEADER = /*"currency converter by /u/Tjstretchalot"*/ "jew boy sandler stalker by /u/Tjstretchalot";

    private long lastRequest;
    @SuppressWarnings("unused")
    private int total_request_count;
    private String myModHash;
    private String myCookie;
    private static final String myFirstTime = "reddit_first=%7B%22firsttime%22%3A%20%22first%22%7D";
    @SuppressWarnings("unused")
    private String user;

    public RedditData() {
        lastRequest = 0;
        total_request_count = 0;
    }

    private static HttpURLConnection createConnection(String urlLoc, int contentlength) throws IOException {
        URL url = new URL(urlLoc);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();

        conn.setDoOutput( contentlength != 0 );
        conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
        conn.setRequestProperty("User-Header", USER_HEADER);
        conn.setRequestProperty("Content-Length", Integer.toString(contentlength));
        conn.setUseCaches (false);
        return conn;
    }

    /**
     * Preferably the connection returned by createconnection
     * @param conn
     * @param addModhash
     * @return
     * @throws IOException
     * @throws RedditException
     */
    private void authorizeConnection( HttpURLConnection conn ) throws IOException, RedditException {
        if(myModHash == null || myCookie == null) {
            throw new RedditException("Attempted authorized method before logging in");
        }
        //System.out.println("[Authorize Connection] Authorizing with the cookie " + myCookie);
        conn.setRequestProperty("Cookie", myCookie);
    }

    private void writeOutput( HttpURLConnection conn, String output ) throws IOException {
        DataOutputStream out = new DataOutputStream(conn.getOutputStream());
        out.writeBytes(output);
        out.close();
    }

    public void login( String username, String password ) throws IOException, RedditException {
        user = username;
        doRequestCheck();

        String urlLoc = "http://www.reddit.com/api/login/" + username;
        String urlParameters = "rem=true&user=" + username + "&passwd=" + password + "&api_type=json";

        HttpURLConnection conn = createConnection( urlLoc, urlParameters.getBytes().length );
        writeOutput(conn, urlParameters);

        JsonReader jr = new JsonReader( new InputStreamReader( conn.getInputStream() ) );

        String modhash = null;
        String cookie = null;
        jr.beginObject();
        String nm;
        while( jr.hasNext() ) {
            nm = jr.nextName();

            if(nm.equals("json")) {
                jr.beginObject();

                while(jr.hasNext()) {
                    nm = jr.nextName();

                    if(nm.equals("data")) {
                        jr.beginObject();

                        while(jr.hasNext()) {
                            nm = jr.nextName();

                            if(nm.equals("modhash")) {
                                modhash = jr.nextString();
                                System.out.println("Found modhash: " + modhash);
                            }else if(nm.equals("cookie")) {
                                cookie = jr.nextString();
                                System.out.println("Found cookie: " + cookie);
                            }else {
                                System.out.println("Found something else: " + nm);
                                jr.skipValue();
                            }
                        }

                        jr.endObject();
                    }else {
                        jr.skipValue();
                    }
                }

                jr.endObject();
            }else {
                jr.skipValue();
            }
        }
        jr.endObject();
        jr.close();

//      BufferedReader in = new BufferedReader( new InputStreamReader( conn.getInputStream() ) );
//
//      System.out.println(in.readLine());

//      in.close();

        System.out.println("Cookie given by json is '" + cookie + "' set-cookie field is '" + conn.getHeaderField("Set-Cookie"));
        cookie = conn.getHeaderField("Set-Cookie");
        cookie = cookie.substring(0, cookie.indexOf(";"));
        conn.disconnect();
        if(modhash == null) {
            throw new RedditException( "Modhash not found" );
        }

        myModHash = modhash;
        myCookie = cookie;


    }

    public List<String> getTopPostURLS( ) throws IOException, RedditException {
        doRequestCheck();
        List<String> result = new ArrayList<>();

        String urlLoc = "http://www.reddit.com/r/all.json";
        HttpURLConnection conn = createConnection( urlLoc, 0 );
        authorizeConnection(conn);
        conn.setRequestProperty("Cookie", conn.getRequestProperty("Cookie") + "; " + myFirstTime);
        JsonReader jr = new JsonReader( new InputStreamReader( conn.getInputStream() ) );

        String nm;


        jr.beginObject();
//      System.out.println("BEGIN OBJECT");
//      printRecursive(jr, jr.nextName(), 2);
//      System.out.println("END OBJECT");
//      jr.endObject();
//      if(1 == 3 / 3)
//          return result;
        while(jr.hasNext()) {
            nm = jr.nextName();
            if(!nm.equals("data") || jr.peek() != JsonToken.BEGIN_OBJECT)
            {
                jr.skipValue();
                continue;
            }

            jr.beginObject();

            while(jr.hasNext()) {
                nm = jr.nextName();
                if(nm.equals("children")) {
                    jr.beginArray();
                    while( jr.hasNext() ) {
                        jr.beginObject();
                        while(jr.hasNext()) {
                            nm = jr.nextName();
                            if(nm.equals("data")) {
                                jr.beginObject();
                                while(jr.hasNext()) {
                                    nm = jr.nextName();
                                    if(nm.equals("permalink")) {
                                        result.add(jr.nextString());
                                    }else {
                                        jr.skipValue();
                                    }
                                }
                                jr.endObject();

                            }else {
                                jr.skipValue();
                            }
                        }
                        jr.endObject();
                    }
                    jr.endArray();
                }else if(nm.equals("modhash")) {
                    myModHash = jr.nextString();
                    System.out.println("Modhash updated; now it is " + myModHash);
                }else {
                    jr.skipValue();
                }
            }
            jr.endObject();
        }
        jr.endObject();
        jr.close();
        conn.disconnect();
        return result;
    }



    private void printRecursive(JsonReader jr, String nm, int spacing) throws IOException {
        String spcString = "";
        for(int i = 0; i < spacing; i++) {
            spcString += " ";
        }
        while(jr.hasNext()) {
            if(jr.peek() == JsonToken.END_ARRAY)
            {
                jr.endArray();
                return;
            }else if(jr.peek() == JsonToken.END_OBJECT) {
                jr.endObject();
                return;
            }else if(jr.peek() == JsonToken.BEGIN_ARRAY) {
                jr.beginArray();
                System.out.println(spcString + "ARRAY " + nm);
                printRecursive(jr, nm, spacing + 2);
                jr.endArray();
            }else if(jr.peek() == JsonToken.BEGIN_OBJECT) {
                jr.beginObject();
                System.out.println(spcString + "OBJECT " + nm);
                printRecursive(jr, nm, spacing + 2);
                jr.endObject();
            }else if(jr.peek() == JsonToken.NAME)
                nm = jr.nextName();
            else {
                System.out.print(spcString + nm + "=");
                switch(jr.peek()) {
                case STRING:
                    System.out.println(jr.nextString().replace("\n", spcString + "\n"));
                    break;
                case NUMBER:
                    System.out.println(jr.nextDouble());
                    break;
                case NULL:
                    System.out.println("null");
                    jr.skipValue();
                    break;
                case BOOLEAN:
                    System.out.println(jr.nextBoolean());
                    break;
                default:
                    System.out.println("Unknown");
                }
            }
        }
    }

    public List<Comment> loadAllComments( String commentlink, List<String> skip_ids, String... regex ) throws IOException, RedditException {
        doRequestCheck();
        HashMap<String, Object> page_data;
        List<Comment> result = new ArrayList<>();

        List<Matcher> matchers = new ArrayList<>();
        for(String r : regex) {
            Pattern p = Pattern.compile(r);
            matchers.add(p.matcher(""));
        }

        String urlLoc = "http://www.reddit.com" + commentlink + ".json"; // http://reddit.com/r/gaming/comments/1405fo/the_humble_thq_bundle/.json for example
        HttpURLConnection conn = createConnection( urlLoc, 0 );
        authorizeConnection(conn);

        String nm;

        JsonReader jr = new JsonReader( new InputStreamReader( conn.getInputStream() ) );

        jr.beginArray();

        page_data = loadPageData(jr);
        jr.beginObject();
        while(jr.hasNext()) {
            nm = jr.nextName();
            if(nm.equals("data")) {
                jr.beginObject();
                while(jr.hasNext()) {
                    nm = jr.nextName();

                    if(nm.equals("children")) {
                        jr.beginArray();

                        while(jr.hasNext()) {
                            jr.beginObject();

                            while(jr.hasNext()) {
                                nm = jr.nextName();

                                if(nm.equals("data")) {
                                    jr.beginObject();

                                    parseCommentIntoList( result, skip_ids, jr, matchers, (String) page_data.get("permalink") );

                                    jr.endObject();
                                }else {
                                    jr.skipValue();
                                }
                            }

                            jr.endObject();
                        }

                        jr.endArray();
                    }else if(nm.equals("modhash")) {
                        myModHash = jr.nextString();
                        System.out.println("Modhash updated; now it is " + myModHash);
                    }else {
                        jr.skipValue();
                    }
                }
                jr.endObject();
            }else {
                jr.skipValue();
            }
        }
        jr.endObject();
        jr.endArray();
        jr.close();
        conn.disconnect();
        return result;
    }

    private HashMap<String, Object> loadPageData(JsonReader jr) throws IOException {
        HashMap<String, Object> page_data = new HashMap<>();
        jr.beginObject();
        String nm;
        while(jr.hasNext()) {
            nm = jr.nextName();

            if(nm.equals("data")) {
                jr.beginObject();

                while(jr.hasNext()) {
                    nm = jr.nextName();

                    if(nm.equals("children")) {
                        jr.beginArray();
                        while(jr.hasNext()) {
                            nm = jr.nextName();

                            JsonToken type = jr.peek();
                            switch(type) {
                            case STRING:
                                page_data.put(nm, jr.nextString());
                                break;
                            case NUMBER:
                                page_data.put(nm,  jr.nextDouble());
                                break;
                            case NULL:
                                page_data.put(nm, null);
                                break;
                            case BOOLEAN:
                                page_data.put(nm, jr.nextBoolean());
                                break;
                            default:
                                jr.skipValue();
                                break;
                            }
                        }
                        jr.endArray();
                    }else {
                        jr.skipValue();
                    }
                }

                jr.endObject();
            }else {
                jr.skipValue();
            }
        }

        jr.endObject();
        return page_data;
    }

    private void parseCommentIntoList(List<Comment> result, List<String> skip_ids, JsonReader jr, List<Matcher> matchers, String oppermalink) throws IOException {
        Comment base = new Comment();
        base.oppermalink = oppermalink;
        int ups = 0;
        int downs = 0;
        while(jr.hasNext()) {
            String nm = jr.nextName();


            if(nm.equals("author")) {
                base.author = jr.nextString();
                //System.out.println("Found author name: " + base.author);
            }else if(nm.equals("link_title"))  {
                base.title = jr.nextString();
            }else if(nm.equals("name")) {
                base.permalink = jr.nextString();
                //System.out.println("Found id: " + base.permalink);
            }else if(nm.equals("body")) {
                base.data = jr.nextString();
                //System.out.println("Found body: " + base.data);
            }else if(nm.equals("link_id")) {
                base.opid = jr.nextString();
            }else if(nm.equals("id")) {
                base.linkid = jr.nextString();
            }else if(nm.equals("ups")) {
                ups = jr.nextInt();
            }else if(nm.equals("downs")) {
                downs = jr.nextInt();
            }else if(nm.equals("subreddit")) {
                base.subreddit = jr.nextString();
            }else if(nm.equals("replies")) {
                //System.out.println("Found replies on comment; is the comment complete? " + base.isCompleteComment() );
                if(jr.peek() != JsonToken.BEGIN_OBJECT) {
                    //System.out.println("Found odd value: " + jr.nextString());
                    jr.skipValue();
                    continue;
                }
                jr.beginObject();
                while( jr.hasNext() ) {
                    nm = jr.nextName();

                    if(nm.equals("children")) {
                        jr.beginArray();

                        while(jr.hasNext()) {
                            jr.beginObject();

                            while(jr.hasNext()) {
                                nm = jr.nextName();

                                if(nm.equals("data")) {
                                    jr.beginObject();
                                    //System.out.println("NEW COMMENT");
                                    parseCommentIntoList(result, skip_ids, jr, matchers, oppermalink);
                                    //System.out.println("END");
                                    jr.endObject();
                                }else {
                                    jr.skipValue();
                                }
                            }

                            jr.endObject();
                        }

                        jr.endArray();
                    }else {
                        jr.skipValue();
                    }
                }
                jr.endObject();
            }else {
                jr.skipValue();
            }
        }

        if(!base.isCompleteComment() || skip_ids.contains(base.permalink) || base.author.equals("ConvertsCurrencyBot")) {
            return;
        }
        base.points = ups - downs;
        for(Matcher m : matchers) {
            m.reset(base.data);

            if(m.find()) {
                result.add(base);
                return;
            }
        }
    }

    public void postComment(String txt, String parentID) throws IOException, RedditException {
        doRequestCheck();
        String urlLoc = "http://www.reddit.com/api/comment";
        String urlParameters = "parent=" + parentID + "&text=" + txt + "&uh=" + myModHash + "&api_type=json";
        //System.out.println(urlParameters + ", byte length is " + urlParameters.getBytes().length);
        HttpURLConnection conn = createConnection(urlLoc, urlParameters.getBytes().length);
        authorizeConnection(conn);
        writeOutput(conn, urlParameters);
        BufferedReader in = new BufferedReader( new InputStreamReader( conn.getInputStream() ) );

        String str;
        while((str = in.readLine()) != null) {
            System.out.println(str);
        }

        in.close();

        conn.disconnect();
        increaseDelay( 1000 * 60 * 10 ); // 15 minutes in between comments
    }

    private void doRequestCheck() throws TooManyRedditRequests {
        if( !canMakeNewRequest() )
            throw new TooManyRedditRequests( "You have made too many requests recently" );
        lastRequest = System.currentTimeMillis();
        total_request_count++;
    }

    public boolean canMakeNewRequest() {
        return System.currentTimeMillis() - lastRequest > MIN_REQUEST_DELAY;
    }

    public void increaseDelay(int i) {
        lastRequest += i;
    }

    public List<Comment> loadUserComments(String userN, List<String> skip_ids, List<Matcher> matchers) throws TooManyRedditRequests, IOException {
        doRequestCheck();

        List<Comment> res = new ArrayList<>();

        String URL = "http://reddit.com/user/" + userN + ".json";
        HttpURLConnection conn = createConnection(URL, 0);

        JsonReader in = new JsonReader(new InputStreamReader(conn.getInputStream()));
        in.setLenient(false);
        String nm;
        in.beginObject();
        while(in.hasNext()) {
            nm = in.nextName();
            System.out.println(nm);
            if(nm.equals("data")) {
                in.beginObject();
                while(in.hasNext()) {
                    nm = in.nextName();

                    if(nm.equals("children")) {
                        in.beginArray();

                        while(in.hasNext()) {
                            in.beginObject();

                            while(in.hasNext()) {
                                nm = in.nextName();

                                if(nm.equals("data")) {
                                    in.beginObject();
                                    parseCommentIntoList(res, skip_ids, in, matchers, "NOT_FOUND");
                                    in.endObject();
                                }else {
                                    in.skipValue();
                                }
                            }

                            in.endObject();
                        }

                        in.endArray();
                    }else {
                        in.skipValue();
                    }
                }

                in.endObject();
            }else if(nm.equals("error")) {
                System.err.println("Found an error! Printing the rest recursively.");
                printRecursive(in, nm, 2);
            }else {
                in.skipValue();
            }
        }
        in.endObject();
        in.close();
        return res;
    }

    public void findPermaLink(Comment c) throws IOException, RedditException {
        if(c == null)
            return;
        doRequestCheck();

        HttpURLConnection conn = createConnection("http://www.reddit.com/by_id/" + c.opid + ".json", 0);
        authorizeConnection(conn);
        //java.io.FileReader f = new java.io.FileReader(new java.io.File("byid.txt"));

        JsonReader in = new JsonReader(new InputStreamReader(conn.getInputStream()));
        //JsonReader in = new JsonReader(f);
        HashMap<String, Object> data = new HashMap<>();
        String nm;
        in.beginObject();
        while(in.hasNext()) {
            nm = in.nextName();
            if(nm.equals("data")) {
                //System.out.println("1");
                in.beginObject();
                while(in.hasNext()) {
                    nm = in.nextName();

                    if(nm.equals("children")) {
                        //System.out.println("2");
                        in.beginArray();
                        in.beginObject();

                        while(in.hasNext()) {
                            nm = in.nextName();

                            if(nm.equals("data")) {
                                //System.out.println("3");
                                in.beginObject();
                                while(in.hasNext()) {
                                    nm = in.nextName();

                                    JsonToken type = in.peek();
                                    switch(type) {
                                    case STRING:
                                        data.put(nm, in.nextString());
                                        break;
                                    case NUMBER:
                                        data.put(nm,  in.nextDouble());
                                        break;
                                    case NULL:
                                        data.put(nm, null);
                                        in.skipValue();
                                        break;
                                    case BOOLEAN:
                                        data.put(nm, in.nextBoolean());
                                        break;
                                    default:
//                                      System.out.println("Skipping a " + type);
                                        in.skipValue();
//                                      System.out.println("Still have stuff: " + in.hasNext());
                                        break;
                                    }
                                }
                                in.endObject();
                            }else
                                in.skipValue();
                        }

                        in.endObject();
                        in.endArray();
                    }else {
                        in.skipValue();
                    }
                }
                in.endObject();
            }else {
                in.skipValue();
            }

        }
        in.endObject();
        in.close();

        c.oppermalink = (String) data.get("url");
        //System.out.println(data);

    }
}