Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- return loads(fp.read(),
- AttributeError: 'int' object has no attribute 'read'
- python twitter.py
- __author__ = 'Tom Cusack'
- import pymongo
- import oauth2 as oauth
- import urllib2, json
- import sys, argparse, time
- def oauth_header(url, consumer, token):
- params = {'oauth_version': '1.0',
- 'oauth_nonce': oauth.generate_nonce(),
- 'oauth_timestamp': int(time.time()),
- }
- req = oauth.Request(method = 'GET',url = url, parameters = params)
- req.sign_request(oauth.SignatureMethod_HMAC_SHA1(),consumer, token)
- return req.to_header()['Authorization'].encode('utf-8')
- def main():
- ### Twitter Settings
- numtweets = '32000'
- verbose = 'store_true'
- retweet = 'store_false'
- CONSUMER_KEY = 'M7Xu9Wte0eIZvqhb4G9HnIn3G'
- CONSUMER_SECRET = 'c8hB4Qwps2aODQUx7UsyzQuCRifEp3PKu6hPQll8wnJGIhbKgZ'
- ACCESS_TOKEN = '3213221313-APuXuNjVMbRbZpu6sVbETbgqkponGsZJVT53QmG'
- ACCESS_SECRET = 'BJHrqWC9ed3pA5oDstSMCYcUcz2pYF3DmJ7jcuDe7yxvi'
- base_url = url = 'https://api.twitter.com/1.1/search/tweets.json?include_entities=true&count=200&q=#mongodb&include_rts=%s' % (retweet)
- oauth_consumer = oauth.Consumer(key = CONSUMER_KEY, secret = CONSUMER_SECRET)
- oauth_token = oauth.Token(key = ACCESS_TOKEN, secret = ACCESS_SECRET)
- ### Mongodb Settings
- uri = 'mongodb://127.0.0.1:27017/SARKY'
- if uri != None:
- try:
- conn = pymongo.MongoClient(uri)
- print 'Pulling Tweets..'
- except:
- print 'Error: Unable to connect to DB. Check uri variable.'
- return
- uri_parts = pymongo.uri_parser.parse_uri(uri)
- db = conn[uri_parts['database']]
- db['twitter-harvest'].ensure_index('id_str')
- ### Helper Variables for Harvest
- max_id = -1
- tweet_count = 0
- stream = 0
- ### Begin Harvesting
- while True:
- auth = oauth_header(url, oauth_consumer, oauth_token)
- headers = {"Authorization": auth}
- request = urllib2.Request(url, headers = headers)
- try:
- stream = urllib2.urlopen(request)
- except urllib2.HTTPError, err:
- if err.code == 404:
- print 'Error: Unknown user. Check --user arg'
- return
- if err.code == 401:
- print 'Error: Unauthorized. Check Twitter credentials'
- return
- tweet_list = json.load(stream)
- if len(tweet_list) == 0:
- print 'No tweets to harvest!'
- return
- if 'errors' in tweet_list:
- print 'Hit rate limit, code: %s, message: %s' % (tweets['errors']['code'], tweets['errors']['message'])
- return
- if max_id == -1:
- tweets = tweet_list
- else:
- tweets = tweet_list[1:]
- if len(tweets) == 0:
- print 'Finished Harvest!'
- return
- for tweet in tweets:
- max_id = id_str = tweet['id_str']
- try:
- if tweet_count == numtweets:
- print 'Finished Harvest- hit numtweets!'
- return
- if uri != None:
- db[user].update({'id_str':id_str},tweet,upsert = True)
- else:
- print tweet['text']
- tweet_count+=1
- if verbose == True and uri != None:
- print tweet['text']
- except Exception, err:
- print 'Unexpected error encountered: %s' %(err)
- return
- url = base_url + '&max_id=' + max_id
- if __name__ == '__main__':
- try:
- main()
- except SystemExit as e:
- if e.code == 0:
- pass
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement