View difference between Paste ID: Mnmfs7zN and

-
+import threading
 import json
 import fileinput
 import os
 import sys
 import time
 import re
 from collections import defaultdict
 class Indexer ():
 	dictionary = defaultdict(list)
 	categories = defaultdict(list)
 	users = defaultdict(list)
 	tweets = []
 	tweet_counts = 0
 	tweets_dir = '../data/tweets/';
 	categories_dir = '../data/categories/';
 	users_dir = '../data/users/';
 	dictionary_dir = '../data/dictionary/';
 
 	def run (self):
 		self.index()
 	def index(self):
 		for line in fileinput.input(['../data/raw_tweets.txt']):
 			if self.tweet_counts % 1000 == 0:
 				self.save_data(self.users, self.users_dir)
 				self.save_data(self.categories, self.categories_dir)
 				self.save_data(self.dictionary, self.dictionary_dir)
 			self.tweet_counts += 1
 			tweet_array = line.split(",")
 
 			#get rid of follower data
 			tweet_array.pop(-1)
 			tweet_array.pop(-1)
 			user_name = tweet_array.pop(-1)
 			display_user_name = user_name
 			user_name = user_name.lower()
 
 			#get rid of timestamp
 			tweet_array.pop(0)
 
 			#put the tweet back together, incase the actual message contained commas
 			tweet = ','.join(tweet_array)
 
 			tweet_id = self.tweet_counts
 			self.users[user_name].append(tweet_id)
 			clean_tweet = re.compile('[^a-zA-Z0-9@#]').sub(' ', tweet)
 			tweet = self.tokenize_line(tweet_id, clean_tweet)
 			tweet = (user_name + '%' + tweet).replace(user_name, '<a href = "/index.php?query=' + user_name + '&type=u">' + display_user_name + '</a>');
 			self.tweets.append(tweet)
 
 	def tokenize_line(self, tweet_id, tweet):
 		tokens = tweet.split(" ")
 		for token in tokens:
 			if len(token) == 0:
 				continue
 			display_token = token
 			#index the token after putting it to lower case, this ensures, that case-conflicts don't emerge when fetching said token
 			token = token.lower()
 			file_name = self.get_file_name(token, 0, 1)
 
 			#replace the category with a link to the categories
 			if token[0] == "#":
 				token = token[1:len(token)-1]
 				tweet = tweet.replace(display_token, '<a href = "/index.php?query=' + token + '&type=c">' + display_token + '</a>');
 				self.categories[token].append(tweet_id)
 
 			#make the username clickable
 			elif token[0] == "@":
 				token_display = token
 				token = token[1:len(token)-1]
 				tweet = tweet.replace(display_token, '<a href = "/index.php?query=' + token + '&type=u">' + display_token + '</a>');
 
 			self.dictionary[token].append(tweet_id)
 		return tweet
 
 	def get_file_name(self, token, lower, upper):
 		result = ""
 		upper += 1
 		if len(token) >= (upper):
 			result = token[lower:upper]
 		elif len(token) >= (lower):
 			result = token[lower]
 		return result
 
 	#saves the hash content to the given file after
 	#loading any existing data
 	def save_data(self, hash, directory):
 		for token in hash:
 			if len(token) == 0:
 				continue
 			id_list = hash[token]
 			file_name = self.get_file_name(token, 0, 1)
 			file_path = directory + file_name + '.txt'
 			file = self.open_file(file_path)
 			existing_tweets = self.read_file(file)
 			if existing_tweets.get(token, False) == False:
 				existing_tweets[token] = []
 			existing_tweets[token].extend(id_list)
 			sorted(existing_tweets[token])
 			file.close()
 
 			#open file handler to write
 			file = open(file_path, 'w')
 			file.write(json.dumps(existing_tweets))
 			file.close()
 		hash = defaultdict(list)
 
 	#returns a file object for a file. if it doesn't exist, it makes it
 	def open_file(self, file):
 		if os.path.exists(file):
 			return open(file, 'r')
 		open(file, 'w+').close()
 		return open(file, 'r')
 
 	def read_file(self, file):
 		file_contents = file.read()
 		if len(file_contents) == 0:
 			return {}
 		else:
 			return json.loads(file_contents)
 
 
 
 def main():
     i = Indexer()
     start = time.time()
     i.index()
     end = time.time()
     elapsed= end - start
     print (elapsed)
 
 if __name__ == "__main__":
     main()