import threading
import json
import fileinput
import os
import sys
import time
import re
from collections import defaultdict
class Indexer ():
	dictionary = defaultdict(list)
	categories = defaultdict(list)
	users = defaultdict(list)
	tweets = []
	tweet_counts = 0
	tweets_dir = '../data/tweets/';
	categories_dir = '../data/categories/';
	users_dir = '../data/users/';
	dictionary_dir = '../data/dictionary/';

	def run (self):
		self.index()
	def index(self):
		for line in fileinput.input(['../data/raw_tweets.txt']):
			if self.tweet_counts % 1000 == 0:
				self.save_data(self.users, self.users_dir)
				self.save_data(self.categories, self.categories_dir)
				self.save_data(self.dictionary, self.dictionary_dir)
			self.tweet_counts += 1
			tweet_array = line.split(",")
			
			#get rid of follower data
			tweet_array.pop(-1)
			tweet_array.pop(-1)
			user_name = tweet_array.pop(-1)
			display_user_name = user_name
			user_name = user_name.lower()
			
			#get rid of timestamp
			tweet_array.pop(0)
			
			#put the tweet back together, incase the actual message contained commas 
			tweet = ','.join(tweet_array)
			
			tweet_id = self.tweet_counts
			self.users[user_name].append(tweet_id)
			clean_tweet = re.compile('[^a-zA-Z0-9@#]').sub(' ', tweet)
			tweet = self.tokenize_line(tweet_id, clean_tweet)
			tweet = (user_name + '%' + tweet).replace(user_name, '<a href = "/index.php?query=' + user_name + '&type=u">' + display_user_name + '</a>');
			self.tweets.append(tweet)
		
	def tokenize_line(self, tweet_id, tweet):
		tokens = tweet.split(" ")
		for token in tokens:
			if len(token) == 0:
				continue
			display_token = token
			#index the token after putting it to lower case, this ensures, that case-conflicts don't emerge when fetching said token
			token = token.lower()
			file_name = self.get_file_name(token, 0, 1)
			
			#replace the category with a link to the categories
			if token[0] == "#":
				token = token[1:len(token)-1]
				tweet = tweet.replace(display_token, '<a href = "/index.php?query=' + token + '&type=c">' + display_token + '</a>');
				self.categories[token].append(tweet_id)	
				
			#make the username clickable 
			elif token[0] == "@":
				token_display = token
				token = token[1:len(token)-1]
				tweet = tweet.replace(display_token, '<a href = "/index.php?query=' + token + '&type=u">' + display_token + '</a>');
				
			self.dictionary[token].append(tweet_id)
		return tweet
	
	def get_file_name(self, token, lower, upper):
		result = ""
		upper += 1
		if len(token) >= (upper):
			result = token[lower:upper]
		elif len(token) >= (lower):
			result = token[lower]
		return result
	
	#saves the hash content to the given file after
	#loading any existing data 
	def save_data(self, hash, directory):
		for token in hash:
			if len(token) == 0:
				continue
			id_list = hash[token]
			file_name = self.get_file_name(token, 0, 1)
			file_path = directory + file_name + '.txt'
			file = self.open_file(file_path)
			existing_tweets = self.read_file(file)
			if existing_tweets.get(token, False) == False:
				existing_tweets[token] = []
			existing_tweets[token].extend(id_list)
			sorted(existing_tweets[token])
			file.close()
			
			#open file handler to write
			file = open(file_path, 'w')
			file.write(json.dumps(existing_tweets))
			file.close()
		hash = defaultdict(list)
	
	#returns a file object for a file. if it doesn't exist, it makes it
	def open_file(self, file):
		if os.path.exists(file):
			return open(file, 'r')
		open(file, 'w+').close()
		return open(file, 'r')
	
	def read_file(self, file):
		file_contents = file.read()
		if len(file_contents) == 0:
			return {}
		else:
			return json.loads(file_contents)
			
			
		
		
			
i = Indexer()
start = time.time()
i.index()
end = time.time()
elapsed= end - start
print (elapsed)