Guest User

Untitled

a guest
Jan 16th, 2018
67
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.65 KB | None | 0 0
  1. // Create Docker
  2. // Run Docker
  3. // Grab Twitter
  4. // save to firebase
  5. // send link
  6. // shut it down
  7.  
  8. const TwitterCrawler = require('twitter-crawler');
  9. const fs = require('fs');
  10. const log = require('winston');
  11. const Promise = require('bluebird');
  12.  
  13. const getEnvCredentials = () =>
  14. process.env.TWITTER_CREDENTIALS ? JSON.parse(process.env.TWITTER_CREDENTIALS) : [];
  15.  
  16. const credentials = [{
  17. consumer_key: '...',
  18. consumer_secret: '...',
  19. access_token_key: '...',
  20. access_token_secret: '...',
  21. enabled: true
  22. }].concat(getEnvCredentials());
  23.  
  24.  
  25. const bind = (object, method) => object[method].bind(object);
  26.  
  27. function saveOutput(obj, filename) {
  28. fs.writeFile(`${__dirname}/output/${filename}`, JSON.stringify(obj, null, ' '));
  29. }
  30.  
  31. const crawler = new TwitterCrawler(credentials);
  32. const crawlList = ["DataCamp"];
  33.  
  34. module.exports = Promise.all(crawlList.map((twitterHandle) => {
  35. // Get user
  36. log.info(`Obtaining user with id ${twitterHandle}...`);
  37. return crawler.getUser(twitterHandle)
  38. .then((user) => {
  39. log.info(
  40. `Obtained info for user ${user.name} (${user.id}). ` +
  41. `Storing in output/${twitterHandle}_user.json`
  42. );
  43. saveOutput(user, `${twitterHandle}_user.json`);
  44. // Crawl tweets
  45. log.info('Obtaining tweets...');
  46. return crawler.getTweets(twitterHandle, {})
  47. .then((tweets) => {
  48. log.info(
  49. `Obtained ${tweets.length} tweets for user ${user.name} (${user.id}). ` +
  50. `Storing in output/${twitterHandle}_tweets.json`
  51. );
  52.  
  53. saveOutput(tweets, `${twitterHandle}_tweets.json`);
  54. log.info('Crawling finished.');
  55. });
  56. })
  57. .catch(bind(log, 'error'));
  58. }));
Add Comment
Please, Sign In to add comment