import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib.cm as cm
from matplotlib import offsetbox

from sklearn.cluster import MiniBatchKMeans
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE


document = ['incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete','incident','install','delete']

sizelist = len(document)
tfidf = TfidfVectorizer(
    min_df = 5,
    max_df = 0.95,
    max_features = 8000,
    stop_words = 'english'
)
tfidf.fit(document)
text = tfidf.transform(document)




def find_optimal_clusters(data, max_k):
    iters = range(2, max_k+1, 2)

    sse = []
    for k in iters:
        sse.append(MiniBatchKMeans(n_clusters=k, init_size=1024, batch_size=2048, random_state=20).fit(data).inertia_)
        print('Fit {} clusters'.format(k))

    f, ax = plt.subplots(1, 1)
    ax.plot(iters, sse, marker='o')
    ax.set_xlabel('Cluster Centers')
    ax.set_xticks(iters)
    ax.set_xticklabels(iters)
    ax.set_ylabel('SSE')
    ax.set_title('SSE by Cluster Center Plot')

find_optimal_clusters(text, 10)


clusters = MiniBatchKMeans(n_clusters=6, init_size=1024, batch_size=2048, random_state=20).fit_predict(text)

def plot_tsne_pca(data, labels, sizelist):
    max_label = max(labels)
    max_items = np.random.choice(range(data.shape[0]), sizelist, replace=False)

    pca = PCA(n_components=2).fit_transform(data[max_items,:].todense())
    tsne = TSNE().fit_transform(PCA(n_components=1).fit_transform(data[max_items,:].todense()))

    idx = np.random.choice(range(pca.shape[0]), sizelist, replace=False)
    label_subset = labels[max_items]
    label_subset = [cm.hsv(i/max_label) for i in label_subset[idx]]
    f, ax = plt.subplots(1, 2, figsize=(20, 6))

    ax[0].scatter(pca[idx, 0], pca[idx, 1], c=label_subset)
    ax[0].set_title('PCA Cluster Plot')

    sc = ax[1].scatter(tsne[idx, 0], tsne[idx, 1], c=label_subset, cmap='tab10')
    ax[1].set_title('TSNE Cluster Plot')
    ax[1].legend(*sc.legend_elements(), title='clusters')



plot_tsne_pca(text, clusters, sizelist)

def get_top_keywords(data, clusters, labels, n_terms):
    df = pd.DataFrame(data.todense()).groupby(clusters).mean()

    for i,r in df.iterrows():
        print('\nCluster {}'.format(i))
        print(','.join([labels[t] for t in np.argsort(r)[-n_terms:]]))

get_top_keywords(text, clusters, tfidf.get_feature_names(), 6)

plt.show()