PremiumAPI

from searchtweets import gen_rule_payload, load_credentials, collect_results
import InputOutput as io
import time
import math
from datetime import datetime, timedelta
from email.utils import parsedate_tz


def to_datetime(datestring):
    time_tuple = parsedate_tz(datestring.strip())
    dt = datetime(*time_tuple[:6])
    return dt - timedelta(seconds=time_tuple[-1])


premium_search_args = load_credentials(r'D:\Python\Twitter\twitter_keys.yaml',
                                       yaml_key="search_tweets_premium",
                                       env_overwrite=False)


def scrape_tweets(keyword, low, high):
    while low < high:
        num = 0
        results_per_call = 500
        max_results = 500
        pt_rule = keyword + ' lang:en'
        low_string = low.strftime('%Y%m%d%H%M')
        high_string = high.strftime('%Y%m%d%H%M')
        rule = gen_rule_payload(pt_rule=pt_rule, from_date=low_string,
                                to_date=high_string, results_per_call=results_per_call)
        tweets = collect_results(rule, max_results=max_results,
                                 result_stream_args=premium_search_args)
        end = tweets[len(tweets) - 1].created_at_datetime
        for tweet in tweets:
            if tweet.lang != 'en':
                continue
            is_retweet = False
            orig_length = 0
            has_comment = False
            if 'retweeted_status' in tweet:
                is_retweet = True
                if 'extended_tweet' in tweet['retweeted_status']:
                    orig_length = len(
                        tweet['retweeted_status']['extended_tweet']['full_text'])
                else:
                    orig_length = len(tweet['retweeted_status']['text'])
            if orig_length > 0 and orig_length < len(tweet.all_text):
                has_comment = True
            lat = ''
            long = ''
            if tweet.geo_coordinates is not None:
                lat = tweet.geo_coordinates['latitude']
                long = tweet.geo_coordinates['longitude']
            io.output_file(['id', 'date', 'text', 'lat', 'long', 'likes', 'replies', 'retweets', 'is_retweet',
                            'has_comment', 'user_id', 'user_name', 'user_date', 'bio', 'location'],
                           [tweet.id, str(tweet.created_at_datetime), tweet.all_text, lat, long,
                               tweet.favorite_count, tweet['reply_count'], tweet.retweet_count, is_retweet, has_comment,
                               tweet.user_id, tweet.screen_name, str(
                               to_datetime(tweet['user']['created_at'])),
                               tweet.bio, tweet['user']['location']],
                           print_columns=False
                           )
            num += 1
        high = end - timedelta(seconds=1)
        print(num)
        print(high)
        delay = max_results / results_per_call
        time.sleep(delay)

# update key directory in code - good
# update endpoint URL - good
# update start and end date - good
# update results per call and max results - good
# update keyword - good


if __name__ == "__main__":
    keyword = 'fat acceptance'
    scrape_tweets(
        keyword,
        datetime(2010, 1, 1),
        datetime(2010, 11, 4, hour=1, minute=43, second=0)
    )