Advertisement
Guest User

Untitled

a guest
May 12th, 2016
68
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 4.26 KB | None | 0 0
  1. package twitterdownloader;
  2.  
  3. import java.io.FileNotFoundException;
  4. import java.io.PrintWriter;
  5. import java.io.UnsupportedEncodingException;
  6. import java.util.HashSet;
  7. import java.util.Map;
  8. import java.util.Set;
  9. import twitter4j.Query;
  10. import twitter4j.QueryResult;
  11. import twitter4j.RateLimitStatus;
  12. import twitter4j.Status;
  13. import twitter4j.Twitter;
  14. import twitter4j.TwitterException;
  15. import twitter4j.TwitterFactory;
  16. import twitter4j.auth.AccessToken;
  17.  
  18. public class TwitterManag {
  19.  
  20.     private final String consumerKey = "XXXX"; //working credentials
  21.     private final String consumerKeySecret = "XXXXX"; //working credentials
  22.     private final String accessToken = "XXXX"; //working credentials
  23.     private final String accessTokenSecret = "XXX"; //working credentials
  24.     private final Twitter twitter; //Twitter4J Instance
  25.     private final Set<Long> idSet; //set od tweet's ids in order to filter result ro receive only unique tweets
  26.     private long maxID = -1; //holding maxId to filter tweets
  27.     private int alreadyDownloaded = 0; //amount of already downloaded unique tweets
  28.     private boolean isBlocked = false; //is API Rate Limit hit?
  29.     private int waitTimeLeftSeconds = 0; //how much to wait till API is available again
  30.  
  31.     public TwitterManag() {
  32.         twitter = TwitterFactory.getSingleton(); //create Twitter4J Instance
  33.         twitter.setOAuthConsumer(consumerKey, consumerKeySecret);
  34.         AccessToken accessTokenAPI = new AccessToken(accessToken, accessTokenSecret);
  35.         twitter.setOAuthAccessToken(accessTokenAPI);
  36.         idSet = new HashSet();
  37.     }
  38.  
  39.     public void downloadTweets(String query, int ammount, String filePathToSave) throws FileNotFoundException, UnsupportedEncodingException, InterruptedException {
  40.         PrintWriter writer = new PrintWriter(filePathToSave, "UTF-8"); //writting tweets to file
  41.         Query twitterQuery = new Query(query);
  42.         twitterQuery.setCount(100);
  43.         twitterQuery.setLang("en");
  44.  
  45.         downloadData(writer, twitterQuery);
  46.  
  47.         while (alreadyDownloaded != ammount) {
  48.             if (!isBlocked) { //if we are not blocked by API
  49.                 twitterQuery.setMaxId(maxID); //set maxID
  50.                 if (ammount - alreadyDownloaded < 100) { //set count to appopiate ammount
  51.                     twitterQuery.setCount(ammount - alreadyDownloaded);
  52.                 } else {
  53.                     twitterQuery.setCount(100);
  54.                 }
  55.                 downloadData(writer, twitterQuery);
  56.             } else { //if we are blocked
  57.                 System.out.println("BLOCKED, WAITING FOR " + waitTimeLeftSeconds / 60 + " minutess.");
  58.                 if (waitTimeLeftSeconds <= 0)
  59.                     waitTimeLeftSeconds = 1;
  60.                 Thread.sleep(waitTimeLeftSeconds * 1000);
  61.                 isBlocked = false;
  62.             }
  63.  
  64.         }
  65.         writer.close();
  66.     }
  67.  
  68.     private void downloadData(PrintWriter writer, Query twitterQuery) {
  69.         try {
  70.             QueryResult result = twitter.search(twitterQuery);
  71.             for (Status singleTweet : result.getTweets()) {
  72.                 if (!idSet.contains(singleTweet.getId()) && !singleTweet.isRetweet()) { //if tweet is not yet in our small database and it is not retweet
  73.                     if (maxID == -1 || singleTweet.getId() < maxID) //get new maxId
  74.                         maxID = singleTweet.getId() - 1;
  75.                    
  76.                     idSet.add(singleTweet.getId()); //add tweet ID to "database"
  77.                     writer.println(singleTweet.getText());
  78.                     alreadyDownloaded++;
  79.                 }
  80.             }
  81.             System.out.println("Already downloaded: " + alreadyDownloaded);
  82.         } catch (TwitterException ex) {
  83.             if (ex.exceededRateLimitation()) {
  84.                 try {
  85.                     Map<String, RateLimitStatus> rateLimitStatus = twitter.getRateLimitStatus("search");
  86.                     RateLimitStatus searchTweetsRateLimit = rateLimitStatus.get("/search/tweets");
  87.                     isBlocked = true;
  88.                     waitTimeLeftSeconds = searchTweetsRateLimit.getSecondsUntilReset();
  89.                 } catch (TwitterException ex1) {
  90.                     waitTimeLeftSeconds = 15 * 60;
  91.                 }
  92.             }
  93.         }
  94.     }
  95. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement