Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # coding: utf-8
- # Extracting the data
- # ==
- # We proceed to get the data from the arff file.
- # In[276]:
- # Let's load the data
- import numpy as np
- import pandas as pd
- from pandas import DataFrame
- from scipy.io.arff import loadarff
- from sklearn.preprocessing import LabelEncoder
- # printing options
- np.set_printoptions(precision=4)
- np.set_printoptions(suppress=True)
- def arfftosk(filename):
- """Function to convert a .arff file to sklearn-compatible format
- Keyword arguments:
- filename -- the name of the file
- """
- data_raw, metadata_raw = loadarff(filename)
- df = pd.DataFrame(data_raw)
- # numpy notation for multiple columns, select everything but the last column
- X_df = df.iloc[:,:-1]
- # last column is the class
- y_df = df.iloc[:,-1]
- y_encoder = LabelEncoder()
- X = X_df.values
- y = y_encoder.fit_transform(y_df)
- return X,y,y_encoder
- # In[277]:
- X_train,y_train, y_encoder = arfftosk("CE802_Ass_2015_Data.arff")
- y_encoder.classes_
- # ### Array contents
- # * Let's see what we have loaded
- # * Notice how missing data are represented as "nan" values
- # In[278]:
- import math
- # Let's check the missing values
- X_dframe = DataFrame(X_train)
- X_dframe.replace(to_replace=float('nan'), value=np.nan, inplace=True)
- X_dframe.shape
- # Imputation
- # ===
- # * What if we have missing values for some data column?
- # * Not all classifiers/regressors work with missing values
- # * Scikit-learn provides methods for *imputing* missing values into the data
- #
- # (credits: made by Spyros Samothrakis and Luca Citi (regressors))
- # ### Remove missing values
- # First we try to remove all the rows with missing values.
- # In[279]:
- # Let's select the columns to be used as regressors
- regressee = 19
- regressors = range(X_train.shape[1])
- regressors.remove(regressee)
- # In[280]:
- '''
- Let's select the instances that can be used to learn to predict
- missing values from the given column.
- '''
- # np.isfinite creates an nparray with all the rows that don't have missing values
- valid = np.isfinite(X_train[:,regressee])
- X_train_no_missing = X_train[valid]
- y_train_no_missing = y_train[valid]
- X_train_no_missing.shape
- # ### Mean Imputation
- # * Using an Imputer with the defualt values, we can just replace
- # * all occurrences of NaN (the python value) with the mean for
- # * that column.
- # In[281]:
- # Imputation, replace every missing column with the mean
- from sklearn.preprocessing import Imputer
- imp = Imputer()
- X_train_imp_mean = imp.fit_transform(X_train)
- # Show the results in the notebook, notice that there are no NaNs
- # left in the column 19 or the 20th feature.
- mean_dframe = DataFrame(X_train_imp_mean)
- mean_dframe
- # ### Linear Regressor Imputation
- # * We can use linear regression to impute the missing data
- # In[282]:
- # Let's train a linear model!
- from sklearn.linear_model import LinearRegression as lr
- imp = lr()
- # ix returns an array of cross products of the 1D arrays sent to
- # it as parameters. So, the cross product of booleans with all
- # regressors, so: X_Train[False, 0], ..., X_Train[False, 18]
- # This is done for each False in the valid array.
- # So you end up with
- imp.fit(X_train[np.ix_(valid,regressors)], X_train[valid,regressee])
- # ... and use it to predict
- imputed_linear = imp.predict(X_train[:,regressors])
- # In[283]:
- # Let's see how it performs on the data
- get_ipython().magic(u'matplotlib inline')
- import matplotlib.pyplot as plt
- plt.figure(figsize=(10,8))
- # picking the first 50 values to be able to see the result in the plot
- plt.plot(X_train[200:251,regressee])
- plt.plot(imputed_linear[200:251])
- plt.show()
- # In[284]:
- # Let's replace the imputed values in place of the missing value
- X_train_imp_linear = X_train.copy()
- X_train_imp_linear[~valid,regressee] = imputed_linear[~valid]
- # ... and plot the results
- plt.figure(figsize=(8,6))
- plt.plot(X_train_imp_linear[:,regressee])
- plt.show()
- # ### Random Forest Regressor Imputation
- # In[285]:
- # Let's train a linear model!
- from sklearn.ensemble import RandomForestRegressor
- imp = RandomForestRegressor(random_state=0, n_estimators=100)
- imp.fit(X_train[np.ix_(valid,regressors)], X_train[valid,regressee])
- # ... and use it to predict
- imputed_random_forest = imp.predict(X_train[:,regressors])
- # In[286]:
- plt.figure(figsize=(10,8))
- # picking the first 50 values to be able to see the result in the plot
- plt.plot(X_train[200:251,regressee])
- plt.plot(imputed_random_forest[200:251])
- plt.show()
- # In[287]:
- # Let's replace the imputed values in place of the missing value
- X_train_imp_forest = X_train.copy()
- X_train_imp_forest[~valid,regressee] = imputed_random_forest[~valid]
- # ... and plot the results
- plt.figure(figsize=(8,6))
- plt.plot(X_train_imp_forest[:,regressee])
- plt.show()
- # ### Plotting all the imputations to compare them
- # In[288]:
- plt.figure(figsize=(10,10))
- plt.plot(X_train[150:251,regressee])
- plt.plot(imputed_linear[150:251])
- plt.plot(imputed_random_forest[150:251])
- plt.legend(["non", "linear", "random forests"])
- plt.show()
- # ### Comparing imputation methods
- # In order to compare imputation methods, crossvalidation can be used, and their kappa statistic can be observed. Will do this if I have the time.
- # Training a model
- # ===
- # Now that Imputation is done, it is possible to train a model to classify the last variable. This has to be a classifier since the last variable is binary.
- #
- # We are going to use the following
- # 1. Naive Bayes: Simple model to prevent overfitting.
- # 2. Decision Trees: With post prunning to reduce overfitting.
- # 3. Random Forest Classifiers: Make our decision tree model more general.
- #
- # But first, lets setup the global variables that state the k for k-fold cross validation and the max depth for both of the tree algorithms.
- # In[289]:
- cv_folds = 10
- tree_max_depth = None
- # ### Naive Bayes
- # We will use Gaussian Naive Bayes to classify our data first.
- #
- # We will check how it performs on:
- # * Data without missing values X_train_no_missing
- # * Data with mean imputated missing values X_train_imp_mean
- # * Data with imputated mising values with linear regression X_train_imp_linear
- # * Data with imputated missing values with a Random Forest Regressor X_train_imp_forest
- # In[293]:
- from sklearn.naive_bayes import GaussianNB
- from sklearn import cross_validation as cv
- from sklearn import metrics
- clf = GaussianNB()
- # Remove missing values
- bayes_nomissing_pred = cv.cross_val_predict(clf,
- X_train_no_missing, y_train_no_missing,
- cv=cv_folds)
- bayes_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
- bayes_nomissing_pred)
- bayes_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
- bayes_nomissing_pred)
- # Imputation with mean
- bayes_mean_pred = cv.cross_val_predict(clf,
- X_train_imp_mean, y_train,
- cv=cv_folds)
- bayes_mean_acc = metrics.accuracy_score(y_train,
- bayes_mean_pred)
- bayes_mean_kap = metrics.cohen_kappa_score(y_train,
- bayes_mean_pred)
- # Imputation with linear regression
- bayes_linear_pred = cv.cross_val_predict(clf,
- X_train_imp_linear, y_train,
- cv=cv_folds)
- bayes_linear_acc = metrics.accuracy_score(y_train,
- bayes_linear_pred)
- bayes_linear_kap = metrics.cohen_kappa_score(y_train,
- bayes_linear_pred)
- # Imputation with random forest regressor
- bayes_forest_pred = cv.cross_val_predict(clf,
- X_train_imp_forest, y_train,
- cv=cv_folds)
- bayes_forest_acc = metrics.accuracy_score(y_train,
- bayes_forest_pred)
- bayes_forest_kap = metrics.cohen_kappa_score(y_train,
- bayes_forest_pred)
- # No cross validation
- clf.fit(X_train_imp_forest, y_train)
- bayes_pred = clf.predict(X_train_imp_forest)
- bayes_acc = metrics.accuracy_score(y_train, bayes_pred)
- bayes_kap = metrics.cohen_kappa_score(y_train, bayes_pred)
- print "\nNo cross validation + Bayes + Impute with Random Forest Regressor"
- print "Acc: %.4f Kappa: %.4f" % (bayes_acc,
- bayes_kap)
- print metrics.confusion_matrix(y_train, bayes_pred)
- print "\nBayes + Remove missing values"
- print "Acc: %.4f Kappa: %.4f" % (bayes_nomissing_acc,
- bayes_nomissing_kap)
- print metrics.confusion_matrix(y_train_no_missing, bayes_nomissing_pred)
- print "\nBayes + Impute with Mean"
- print "Acc: %.4f Kappa: %.4f" % (bayes_mean_acc,
- bayes_mean_kap)
- print metrics.confusion_matrix(y_train, bayes_mean_pred)
- print "\nBayes + Impute with Linear Regressor"
- print "Acc: %.4f Kappa: %.4f" % (bayes_linear_acc,
- bayes_linear_kap)
- print metrics.confusion_matrix(y_train, bayes_linear_pred)
- print "\nBayes + Impute with Random Forest Regressor"
- print "Acc: %.4f Kappa: %.4f" % (bayes_forest_acc,
- bayes_forest_kap)
- print metrics.confusion_matrix(y_train, bayes_forest_pred)
- # ### Decision Tree
- # In[294]:
- from sklearn.tree import DecisionTreeClassifier
- clf = DecisionTreeClassifier(random_state=0, max_depth=tree_max_depth)
- # Remove missing values
- tree_nomissing_pred = cv.cross_val_predict(clf,
- X_train_no_missing, y_train_no_missing,
- cv=cv_folds)
- tree_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
- tree_nomissing_pred)
- tree_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
- tree_nomissing_pred)
- # Imputation with mean
- tree_mean_pred = cv.cross_val_predict(clf,
- X_train_imp_mean, y_train,
- cv=cv_folds)
- tree_mean_acc = metrics.accuracy_score(y_train,
- tree_mean_pred)
- tree_mean_kap = metrics.cohen_kappa_score(y_train,
- tree_mean_pred)
- # Imputation with linear regression
- tree_linear_pred = cv.cross_val_predict(clf,
- X_train_imp_linear, y_train,
- cv=cv_folds)
- tree_linear_acc = metrics.accuracy_score(y_train,
- tree_linear_pred)
- tree_linear_kap = metrics.cohen_kappa_score(y_train,
- tree_linear_pred)
- # Imputation with random forest regressor
- tree_forest_pred = cv.cross_val_predict(clf,
- X_train_imp_forest, y_train,
- cv=cv_folds)
- tree_forest_acc = metrics.accuracy_score(y_train,
- tree_forest_pred)
- tree_forest_kap = metrics.cohen_kappa_score(y_train,
- tree_forest_pred)
- # No cross validation
- clf.fit(X_train_imp_forest, y_train)
- tree_pred = clf.predict(X_train_imp_forest)
- tree_acc = metrics.accuracy_score(y_train, tree_pred)
- tree_kap = metrics.cohen_kappa_score(y_train, tree_pred)
- print "\nNo cross validation + Decision Tree + Impute with Random Forest Regressor"
- print "Acc: %.4f Kappa: %.4f" % (tree_acc,
- tree_kap)
- print metrics.confusion_matrix(y_train, tree_pred)
- print "\nDecision Tree + Remove missing values"
- print "Acc: %.4f Kappa: %.4f" % (tree_nomissing_acc,
- tree_nomissing_kap)
- print metrics.confusion_matrix(y_train_no_missing, tree_nomissing_pred)
- print "\nDecision Tree + Impute with Mean"
- print "Acc: %.4f Kappa: %.4f" % (tree_mean_acc,
- tree_mean_kap)
- print metrics.confusion_matrix(y_train, tree_mean_pred)
- print "\nDecision Tree + Impute with Linear Regressor"
- print "Acc: %.4f Kappa: %.4f" % (tree_linear_acc,
- tree_linear_kap)
- print metrics.confusion_matrix(y_train, tree_linear_pred)
- print "\nDecision Tree + Impute with Random Forest Regressor"
- print "Acc: %.4f Kappa: %.4f" % (tree_forest_acc,
- tree_forest_kap)
- print metrics.confusion_matrix(y_train, tree_forest_pred)
- # ### Random Forest Classifier
- # In[292]:
- from sklearn.ensemble import RandomForestClassifier
- clf = RandomForestClassifier(n_estimators=100,
- random_state=0,
- max_depth=tree_max_depth)
- # Remove missing values
- forest_nomissing_pred = cv.cross_val_predict(clf,
- X_train_no_missing, y_train_no_missing,
- cv=cv_folds)
- forest_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
- forest_nomissing_pred)
- forest_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
- forest_nomissing_pred)
- # Imputation with mean
- forest_mean_pred = cv.cross_val_predict(clf,
- X_train_imp_mean, y_train,
- cv=cv_folds)
- forest_mean_acc = metrics.accuracy_score(y_train,
- forest_mean_pred)
- forest_mean_kap = metrics.cohen_kappa_score(y_train,
- forest_mean_pred)
- # Imputation with linear regression
- forest_linear_pred = cv.cross_val_predict(clf,
- X_train_imp_linear, y_train,
- cv=cv_folds)
- forest_linear_acc = metrics.accuracy_score(y_train,
- forest_linear_pred)
- forest_linear_kap = metrics.cohen_kappa_score(y_train,
- forest_linear_pred)
- # Imputation with random forest regressor
- forest_forest_pred = cv.cross_val_predict(clf,
- X_train_imp_forest, y_train,
- cv=cv_folds)
- forest_forest_acc = metrics.accuracy_score(y_train,
- forest_forest_pred)
- forest_forest_kap = metrics.cohen_kappa_score(y_train,
- forest_forest_pred)
- # No cross validation
- clf.fit(X_train_imp_forest, y_train)
- forest_pred = clf.predict(X_train_imp_forest)
- forest_acc = metrics.accuracy_score(y_train, forest_pred)
- forest_kap = metrics.cohen_kappa_score(y_train, forest_pred)
- print "\nNo cross validation + Random Forest Classifier + Impute with Random Forest Regressor"
- print "Acc: %.4f Kappa: %.4f" % (forest_acc,
- forest_kap)
- print metrics.confusion_matrix(y_train, forest_pred)
- print "\nRandom Forest Classifier + Remove missing values"
- print "Acc: %.4f Kappa: %.4f" % (forest_nomissing_acc,
- forest_nomissing_kap)
- print metrics.confusion_matrix(y_train_no_missing,
- forest_nomissing_pred)
- print "\nRandom Forest Classifier + Impute with Mean"
- print "Acc: %.4f Kappa: %.4f" % (forest_mean_acc,
- forest_mean_kap)
- print metrics.confusion_matrix(y_train, forest_mean_pred)
- print "\nRandom Forest Classifier + Impute with Linear Regressor"
- print "Acc: %.4f Kappa: %.4f" % (forest_linear_acc,
- forest_linear_kap)
- print metrics.confusion_matrix(y_train, forest_linear_pred)
- print "\nRandom Forest Classifier + Impute with Random Forest Regressor"
- print "Acc: %.4f Kappa: %.4f" % (forest_forest_acc,
- forest_forest_kap)
- print metrics.confusion_matrix(y_train, forest_forest_pred)
Add Comment
Please, Sign In to add comment