tweetParser

import de.bezier.data.sql.*;
import http.requests.*;
import java.util.regex.*;

SQLite db;
int imageIndex, userNameIndex;
long id;
String user;
String notExisting = "";
Pattern urlPattern;
Matcher m;
PrintWriter output;
int count = 0;
int tweetImgCount = 0;
int notExistingCount = 0;
int successCount = 0;

void setup() {
  output = createWriter("tweetData.txt");
  urlPattern = Pattern.compile(
  "(?:^|[\\W])((ht|f)tp(s?):\\/\\/|www\\.)"
    + "(([\\w\\-]+\\.){1,}?([\\w\\-.~]+\\/?)*"
    + "[\\p{Alnum}.,%_=?&#\\-+()\\[\\]\\*$~@!:/{};']*)",
  Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL);

  db = new SQLite( this, "../medtek2017kode/data/tweets1.db" );
  if ( db.connect() ) {

    String Q = "select distinct id, user from tweet limit 10";
    db.query(Q);
    String urlMatcher;

    while (db.next ()) {
      count++;
      user = db.getString("user");
      id = db.getLong("id");

      String imgUrl = null;
      String imgContent = null;
      String request = "https://twitter.com/"+user+"/status/"+id;
      Boolean retweet = false;

      //Send request til twitter.com/{<brugernavn>}/status/{<id>}
      GetRequest get = new GetRequest(request);
      get.send();

      imageIndex = get.getContent().indexOf("<div class=\"AdaptiveMedia-photoContainer js-adaptive-photo");
      userNameIndex = get.getContent().indexOf("data-screen-name=\"");

      if (imageIndex != -1) {
        tweetImgCount++;
        imgContent = get.getContent().substring(imageIndex, imageIndex+200);
        //println(imgContent);
        m = urlPattern.matcher(imgContent);
        while (m.find ()) {
          int matchStart = m.start(1);
          int matchEnd = m.end();
          imgUrl = imgContent.substring(matchStart, matchEnd);
        }
      }
      print("---------------\nAll Count:", count, "With img:", tweetImgCount, " | ");
      if (userNameIndex != -1) {
        imgContent = get.getContent().substring(userNameIndex, userNameIndex+200);
        if (!imgContent.contains(user))
          retweet = true;

        println(user, id);
        println("Request:", request);
        if (imgUrl != null)
          println("ImgUrl: ", imgUrl, "\n"+"Retweet/response:", retweet);
      } else {
        notExisting += "("+user+", "+id+ "), ";
        println("\n"+"Not existing:", ++notExistingCount);
      }
      if (imageIndex != -1 && userNameIndex != -1 && !retweet) {
        println("Added:", ++successCount);
        output.println("INSERT INTO image VALUES ("+ id +", '"+imgUrl+"');");
      } else println();
    }

    //println("User "+count+": "+user+"\n    Profile img: "+ ppUrl + "\n    Tweets: " +tweets + "\n    Following: " +following + "\n    Followers: " +followers  + "\n    Likes: " +likes  + "\n    Description: " +description);
    //println("Not existing: "+ notExisting +"\n");
  }
  output.flush(); // Writes the remaining data to the file
  output.close();
  println("\n"+"Not existing: ", notExisting);

  int x = millis()/1000;
  int seconds = x % 60;
  x /= 60;
  int minutes = x % 60;
  x /= 60;
  int hours = x % 24;
  x /= 24;
  int days = x;
  println("Done in", x, "days", hours, "hours", minutes, "minutes and", seconds, "seconds"  );
  exit();
}