#bow representations for the three sets unlabelled, train and test
vectorizer = CountVectorizer(max_features=3000,stop_words='english')


corpus_tfidf_unsuper = vectorizer.fit_transform(train_data_unsupervised[:,2])
corpus_tfidf_train = vectorizer.transform(train_ds[:,2])
corpus_tfidf_test= vectorizer.transform(test_ds[:,2])

#transform to gensim acceptable objects
vocab = vectorizer.get_feature_names()
id2word_unsuper=dict([(i, s) for i, s in enumerate(vocab)])
corpus_vect_gensim_unsuper = matutils.Sparse2Corpus(corpus_tfidf_unsuper.T)
corpus_vect_gensim_train = matutils.Sparse2Corpus(corpus_tfidf_train.T)
corpus_vect_gensim_test = matutils.Sparse2Corpus(corpus_tfidf_test.T)

#fit the model to the unlabelled data
lda = models.LdaModel(corpus_vect_gensim_unsuper,
                  id2word = id2word_unsuper,
                  num_topics = 10,
                  passes=1)
#transform the train and test set to the latent topic space
docTopicProbMat_train = lda[corpus_vect_gensim_train]
docTopicProbMat_test = lda[corpus_vect_gensim_test]
#transform to csr matrices
train_lda=matutils.corpus2csc(docTopicProbMat_train)
test_lda=matutils.corpus2csc(docTopicProbMat_test)
#fit the classifier and print the accuracy
clf =LogisticRegression()
clf.fit(train_lda.transpose(), np.array(train_ds[:,0]).astype(int))
ypred = clf.predict(test_lda.transpose())
print accuracy_score(test_ds[:,0].astype(int), ypred)