Advertisement
Guest User

Untitled

a guest
Apr 28th, 2015
607
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.58 KB | None | 0 0
  1. return loads(fp.read(),
  2. AttributeError: 'int' object has no attribute 'read'
  3.  
  4. python twitter.py
  5.  
  6. __author__ = 'Tom Cusack'
  7.  
  8. import pymongo
  9. import oauth2 as oauth
  10. import urllib2, json
  11. import sys, argparse, time
  12.  
  13. def oauth_header(url, consumer, token):
  14.  
  15. params = {'oauth_version': '1.0',
  16. 'oauth_nonce': oauth.generate_nonce(),
  17. 'oauth_timestamp': int(time.time()),
  18. }
  19. req = oauth.Request(method = 'GET',url = url, parameters = params)
  20. req.sign_request(oauth.SignatureMethod_HMAC_SHA1(),consumer, token)
  21. return req.to_header()['Authorization'].encode('utf-8')
  22.  
  23. def main():
  24. ### Twitter Settings
  25. numtweets = '32000'
  26. verbose = 'store_true'
  27. retweet = 'store_false'
  28. CONSUMER_KEY = 'M7Xu9Wte0eIZvqhb4G9HnIn3G'
  29. CONSUMER_SECRET = 'c8hB4Qwps2aODQUx7UsyzQuCRifEp3PKu6hPQll8wnJGIhbKgZ'
  30. ACCESS_TOKEN = '3213221313-APuXuNjVMbRbZpu6sVbETbgqkponGsZJVT53QmG'
  31. ACCESS_SECRET = 'BJHrqWC9ed3pA5oDstSMCYcUcz2pYF3DmJ7jcuDe7yxvi'
  32. base_url = url = 'https://api.twitter.com/1.1/search/tweets.json?include_entities=true&count=200&q=#mongodb&include_rts=%s' % (retweet)
  33. oauth_consumer = oauth.Consumer(key = CONSUMER_KEY, secret = CONSUMER_SECRET)
  34. oauth_token = oauth.Token(key = ACCESS_TOKEN, secret = ACCESS_SECRET)
  35.  
  36. ### Mongodb Settings
  37. uri = 'mongodb://127.0.0.1:27017/SARKY'
  38. if uri != None:
  39. try:
  40. conn = pymongo.MongoClient(uri)
  41. print 'Pulling Tweets..'
  42. except:
  43. print 'Error: Unable to connect to DB. Check uri variable.'
  44. return
  45. uri_parts = pymongo.uri_parser.parse_uri(uri)
  46. db = conn[uri_parts['database']]
  47. db['twitter-harvest'].ensure_index('id_str')
  48.  
  49. ### Helper Variables for Harvest
  50. max_id = -1
  51. tweet_count = 0
  52. stream = 0
  53.  
  54. ### Begin Harvesting
  55. while True:
  56. auth = oauth_header(url, oauth_consumer, oauth_token)
  57. headers = {"Authorization": auth}
  58. request = urllib2.Request(url, headers = headers)
  59. try:
  60. stream = urllib2.urlopen(request)
  61. except urllib2.HTTPError, err:
  62. if err.code == 404:
  63. print 'Error: Unknown user. Check --user arg'
  64. return
  65. if err.code == 401:
  66. print 'Error: Unauthorized. Check Twitter credentials'
  67. return
  68. tweet_list = json.load(stream)
  69.  
  70. if len(tweet_list) == 0:
  71. print 'No tweets to harvest!'
  72. return
  73. if 'errors' in tweet_list:
  74. print 'Hit rate limit, code: %s, message: %s' % (tweets['errors']['code'], tweets['errors']['message'])
  75. return
  76. if max_id == -1:
  77. tweets = tweet_list
  78. else:
  79. tweets = tweet_list[1:]
  80. if len(tweets) == 0:
  81. print 'Finished Harvest!'
  82. return
  83.  
  84. for tweet in tweets:
  85. max_id = id_str = tweet['id_str']
  86. try:
  87. if tweet_count == numtweets:
  88. print 'Finished Harvest- hit numtweets!'
  89. return
  90. if uri != None:
  91. db[user].update({'id_str':id_str},tweet,upsert = True)
  92. else:
  93. print tweet['text']
  94. tweet_count+=1
  95. if verbose == True and uri != None:
  96. print tweet['text']
  97. except Exception, err:
  98. print 'Unexpected error encountered: %s' %(err)
  99. return
  100. url = base_url + '&max_id=' + max_id
  101.  
  102. if __name__ == '__main__':
  103. try:
  104. main()
  105. except SystemExit as e:
  106. if e.code == 0:
  107. pass
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement