Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/usr/bin/env python
- # -*- coding: utf-8 -*-
- import os
- import sys
- from optparse import OptionParser
- import tweepy
- import time
- import logging
- logger = logging.getLogger(__name__)
- class Streamer(tweepy.StreamListener):
- def __init__(self, *args, **kwargs):
- try:
- self.hashtags = kwargs.pop('hashtags')
- except KeyError:
- pass
- try:
- self.redis_cache = kwargs.pop('redis_cache')
- except KeyError:
- pass
- self.checker = time.time()
- return super(Streamer, self).__init__(*args, **kwargs)
- def on_status(self, status):
- self.process_status(status)
- return self.should_continue()
- def on_error(self, error):
- #print error
- return self.should_continue()
- def on_timeout(self):
- return self.should_continue()
- def should_continue(self):
- if time.time() - self.checker > 120:
- self.checker = time.time()
- return set(x.name for x in Hashtag.objects.filter(profiles__isnull=False)) == set(self.hashtags)
- return True
- def is_good(self, user):
- if user.default_profile_image:
- return False
- if not user.description or 'bot' in user.description:
- return False
- if user.followers_count < 50:
- return False
- if user.friends_count > 250:
- return False
- if int(user.friends_count) / float(user.followers_count) > 3:
- return False
- if not user.name:
- return False
- return True
- def cache_users_id(self, twitter_id, hashtag):
- return self.redis_cache.lpush("{}-user_ids".format(hashtag), "{}".format(twitter_id))
- def cache_users_screen_name(self, screen_name, hashtag):
- return self.redis_cache.lpush("{}-user_screen_names".format(hashtag), "{}".format(screen_name))
- def cache_tweet_id(self, twitter_id, hashtag):
- return self.redis_cache.lpush("{}-tweet_id".format(hashtag), '{}'.format(twitter_id))
- def cache_tweet_text(self, text, hashtag):
- return self.redis_cache.lpush("{}-tweet_text".format(hashtag), "{}".format(text))
- def process_status(self, status):
- if not self.is_good(status.user):
- return True
- else:
- twitter_id = int(status.user.id)
- screen_name = str(status.user.screen_name)
- status_id = int(status.id)
- t_status = str(status.text.encode('utf-8'))
- for x in status.entities['hashtags']:
- hashtag = str(x['text'].lstrip('#').lower().encode('utf-8'))
- self.cache_users_id(hashtag=hashtag, twitter_id=twitter_id)
- self.cache_users_screen_name(hashtag=hashtag, screen_name=screen_name)
- self.cache_tweet_text(text=t_status, hashtag=hashtag)
- class Worker:
- def __init__(self, con):
- apikey = 'moX6M9jbbIuAnYuaAxZJFkzQY'
- apisecret = 'YhH1Fgr4VUyzLsKoKKQrR0bRuPbsqP4daiiZ9UbbSyZWDCsTxU'
- access_token = '1177046514-QVUDUBANp0p2HHiJrBJIwYXyaqjZkQg7NMHSRwA'
- access_secret = 'gBDQ3Z2wNGMOptyAsLffOHksIbhYlL6RvbCYSHzkWV08s'
- self.con = con
- self.auth = tweepy.OAuthHandler(apikey, apisecret)
- self.auth.set_access_token(access_token, access_secret)
- def streamFromTwitter(self):
- while 1:
- hashtags = Hashtag.objects.filter(profiles__isnull=False).distinct()
- if not hashtags:
- time.sleep(10)
- else:
- stream = tweepy.Stream(self.auth, Streamer(hashtags=hashtags, redis_cache=self.con))
- stream.filter(track=['#'+x.name for x in hashtags], languages=['en',])
- if __name__ == '__main__':
- usage = "usage: %prog -s PATH | --path=PATH"
- parser = OptionParser(usage)
- parser.add_option('-s', '--path', dest='path', metavar='PATH', help="The path to the Django environment")
- (options, args) = parser.parse_args()
- if not options.path:
- parser.error("Specify the path where manage.py is")
- os.environ['DJANGO_SETTINGS_MODULE'] = "flockwithme.prod"
- sys.path.append(options.path)
- from django.core.wsgi import get_wsgi_application
- application = get_wsgi_application()
- ####################### IMPORTS ########################
- from django_redis import get_redis_connection
- from django.core.cache import cache
- from flockwithme.app.scheduler.models import TwitterStatus, TwitterUser, Hashtag
- con = get_redis_connection('default')
- ########################################################
- a = Worker(con)
- a.streamFromTwitter()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement