import pandas as pd

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
import numpy as np
from sklearn.preprocessing import LabelEncoder

from keras.models import Sequential
from keras.layers import Embedding, Flatten, Dense
from sklearn.preprocessing import LabelEncoder

df=pd.read_csv('../data/crowdflower_text_emotion.csv')

df.drop(['tweet_id','author'],axis=1,inplace=True)

df=df[~df['sentiment'].isin(['empty','enthusiasm','boredom','anger'])]

df = df.sample(frac=1).reset_index(drop=True)

labels = df['sentiment']
texts = df['content']

tokenizer = Tokenizer(5000)
tokenizer.fit_on_texts(texts)

sequences = tokenizer.texts_to_sequences(texts)

word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))

data = pad_sequences(sequences, maxlen=37)

encoder = LabelEncoder()
encoder.fit(labels)
encoded_Y = encoder.transform(labels)

from keras.utils import np_utils
labels = np_utils.to_categorical(encoded_Y)

print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)

indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
print(labels.shape)

model = Sequential()
model.add(Embedding(5000, 30, input_length=37))
model.add(Flatten())
model.add(Dense(100,activation='relu'))
model.add(Dense(50, activation='relu'))
model.add(Dense(labels.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])

model.fit(data, labels, validation_split=0.2, epochs=10, batch_size=100)