TwitterBot

import tweepy
import csv
import datetime

c_key = "X"
c_secret = "X"

auth = tweepy.AppAuthHandler(c_key, c_secret)
api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True)

num_tweets = 1
prev_num = 1
file_num = -1


class Listener(tweepy.StreamListener):

    def on_status(self, status):
        full_status = ""
        if hasattr(status, "retweeted_status"):
            if hasattr(status.retweeted_status, "extended_tweet"):
                full_status = status.retweeted_status.extended_tweet['full_text']
            else:
                full_status = status.retweeted_status.text
        elif hasattr(status, "extended_tweet"):
            full_status = status.extended_tweet["full_text"]
        else:
            full_status = status.text
        output_file([status.id, status.user.id, str(status.created_at), full_status])

    def on_error(self, status_code):
        debug_out(str(status_code))
        return True


def stream_tweets(keywords):
    stream_listener = Listener()
    stream = tweepy.Stream(auth=api.auth, listener=stream_listener, tweet_mode="extended")
    stream.filter(track=keywords, languages=['en'])


def search_tweets(keywords):
    prev_ids = {}
    for keyword in keywords:
        prev_ids[keyword] = -1
    while True:
        statuses_left = False
        for keyword in keywords:
            statuses = None
            if prev_ids[keyword] >= 0:
                statuses = api.search(q=keyword, tweet_mode="extended", count="100", max_id=str(prev_ids[keyword] - 1))
            else:
                statuses = api.search(q=keyword, tweet_mode="extended", count="100")
            if statuses:
                for status in statuses:
                    full_status = ""
                    if hasattr(status, "retweeted_status"):
                        full_status = status.retweeted_status.full_text
                    else:
                        full_status = status.full_text
                    output_file([status.id, status.user.id, status.created_at, full_status])
                    prev_ids[keyword] = status.id
                    statuses_left = True
        if not statuses_left:
            break


def get_user_tweets(username):
    page = 1
    while True:
        statuses = api.user_timeline(id=username, tweet_mode="extended", count="200", page=page)
        if statuses:
            for status in statuses:
                full_status = ""
                if hasattr(status, "retweeted_status"):
                    full_status = status.retweeted_status.full_text
                else:
                    full_status = status.full_text
                output_file([status.id, status.user.id, status.created_at, full_status])
        else:
            break
        page += 1


def debug_out(value):
    with open("Debug.txt", "w") as debug_out:
        debug_out.write(value + " NumTweets: " + str(num_tweets))
    print(value + " NumTweets: " + str(num_tweets))


def output_file(values):
    global num_tweets
    global prev_num
    global file_num
    new_file = False
    if file_num == -1:
        new_file = True
        file_num = 0
    if num_tweets >= (prev_num + 100000):
        file_num += 1
        prev_num = num_tweets
        new_file = True
    with open("Data" + str(file_num) + ".csv", "a", encoding="utf-8", newline="") as csv_out:
        writer = csv.writer(csv_out, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
        if new_file:
            writer.writerow(["id", "user_id", "date", "text"])
        writer.writerow(values)
        debug_out(str(datetime.datetime.now()))
        num_tweets += 1
        new_file = False


def reader(number):
    for x in range(number):
        with open("Data" + str(x) + ".csv", "r", encoding="utf-8") as csv_in:
            reader = csv.DictReader(csv_in)
            for line in reader:
                print(str(line["id"]))
                print(str(line["user_id"]))
                print(line["date"])
                print(line["text"])
                print()


if __name__ == '__main__':
    keywords = ["gaming", "video games"]
    search_tweets(keywords)
    stream_tweets(keywords)