betoesquivel

Machine learning assignment 1 essex

Jan 14th, 2016
330
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 15.78 KB | None | 0 0
  1.  
  2. # coding: utf-8
  3.  
  4. # Extracting the data
  5. # ==
  6. # We proceed to get the data from the arff file.
  7.  
  8. # In[276]:
  9.  
  10. # Let's load the data
  11. import numpy as np
  12. import pandas as pd
  13. from pandas import DataFrame
  14. from scipy.io.arff import loadarff
  15. from sklearn.preprocessing import LabelEncoder
  16.  
  17. # printing options
  18. np.set_printoptions(precision=4)
  19. np.set_printoptions(suppress=True)
  20.  
  21. def arfftosk(filename):
  22.     """Function to convert a .arff file to sklearn-compatible format
  23.  
  24.    Keyword arguments:
  25.    filename -- the name of the file
  26.    """
  27.  
  28.     data_raw, metadata_raw = loadarff(filename)
  29.     df = pd.DataFrame(data_raw)
  30.     # numpy notation for multiple columns, select everything but the last column
  31.     X_df = df.iloc[:,:-1]
  32.     # last column is the class
  33.     y_df = df.iloc[:,-1]
  34.     y_encoder = LabelEncoder()
  35.     X = X_df.values
  36.     y = y_encoder.fit_transform(y_df)
  37.  
  38.  
  39.     return X,y,y_encoder
  40.  
  41.  
  42. # In[277]:
  43.  
  44. X_train,y_train, y_encoder = arfftosk("CE802_Ass_2015_Data.arff")
  45. y_encoder.classes_
  46.  
  47.  
  48. # ### Array contents
  49. # * Let's see what we have loaded
  50. # * Notice how missing data are represented as "nan" values
  51.  
  52. # In[278]:
  53.  
  54. import math
  55. # Let's check the missing values
  56. X_dframe = DataFrame(X_train)
  57. X_dframe.replace(to_replace=float('nan'), value=np.nan, inplace=True)
  58. X_dframe.shape
  59.  
  60.  
  61. # Imputation
  62. # ===
  63. # * What if we have missing values for some data column?
  64. # * Not all classifiers/regressors work with missing values
  65. # * Scikit-learn provides methods for *imputing* missing values into the data
  66. #
  67. # (credits: made by Spyros Samothrakis and Luca Citi (regressors))
  68.  
  69. # ### Remove missing values
  70. # First we try to remove all the rows with missing values.
  71.  
  72. # In[279]:
  73.  
  74. # Let's select the columns to be used as regressors
  75. regressee = 19
  76. regressors = range(X_train.shape[1])
  77. regressors.remove(regressee)
  78.  
  79.  
  80. # In[280]:
  81.  
  82. '''
  83. Let's select the instances that can be used to learn to predict
  84. missing values from the given column.
  85. '''
  86. # np.isfinite creates an nparray with all the rows that don't have missing values
  87. valid = np.isfinite(X_train[:,regressee])
  88. X_train_no_missing = X_train[valid]
  89. y_train_no_missing = y_train[valid]
  90. X_train_no_missing.shape
  91.  
  92.  
  93. # ### Mean Imputation
  94. # * Using an Imputer with the defualt values, we can just replace
  95. # * all occurrences of NaN (the python value) with the mean for
  96. # * that column.
  97.  
  98. # In[281]:
  99.  
  100. # Imputation, replace every missing column with the mean
  101. from sklearn.preprocessing import Imputer
  102. imp = Imputer()
  103. X_train_imp_mean = imp.fit_transform(X_train)
  104.  
  105. # Show the results in the notebook, notice that there are no NaNs
  106. # left in the column 19 or the 20th feature.
  107. mean_dframe = DataFrame(X_train_imp_mean)
  108. mean_dframe
  109.  
  110.  
  111. # ### Linear Regressor Imputation
  112. # * We can use linear regression to impute the missing data
  113.  
  114. # In[282]:
  115.  
  116. # Let's train a linear model!
  117. from sklearn.linear_model import LinearRegression as lr
  118. imp = lr()
  119. # ix returns an array of cross products of the 1D arrays sent to
  120. # it as parameters. So, the cross product of booleans with all
  121. # regressors, so: X_Train[False, 0], ..., X_Train[False, 18]
  122. # This is done for each False in the valid array.
  123. # So you end up with
  124. imp.fit(X_train[np.ix_(valid,regressors)], X_train[valid,regressee])
  125. # ... and use it to predict
  126. imputed_linear = imp.predict(X_train[:,regressors])
  127.  
  128.  
  129. # In[283]:
  130.  
  131. # Let's see how it performs on the data
  132. get_ipython().magic(u'matplotlib inline')
  133. import matplotlib.pyplot as plt
  134. plt.figure(figsize=(10,8))
  135. # picking the first 50 values to be able to see the result in the plot
  136. plt.plot(X_train[200:251,regressee])
  137. plt.plot(imputed_linear[200:251])
  138. plt.show()
  139.  
  140.  
  141. # In[284]:
  142.  
  143. # Let's replace the imputed values in place of the missing value
  144. X_train_imp_linear = X_train.copy()
  145. X_train_imp_linear[~valid,regressee] = imputed_linear[~valid]
  146. # ... and plot the results
  147. plt.figure(figsize=(8,6))
  148. plt.plot(X_train_imp_linear[:,regressee])
  149. plt.show()
  150.  
  151.  
  152. # ### Random Forest Regressor Imputation
  153.  
  154. # In[285]:
  155.  
  156. # Let's train a linear model!
  157. from sklearn.ensemble import RandomForestRegressor
  158.  
  159. imp = RandomForestRegressor(random_state=0, n_estimators=100)
  160.  
  161. imp.fit(X_train[np.ix_(valid,regressors)], X_train[valid,regressee])
  162. # ... and use it to predict
  163. imputed_random_forest = imp.predict(X_train[:,regressors])
  164.  
  165.  
  166. # In[286]:
  167.  
  168. plt.figure(figsize=(10,8))
  169. # picking the first 50 values to be able to see the result in the plot
  170. plt.plot(X_train[200:251,regressee])
  171. plt.plot(imputed_random_forest[200:251])
  172. plt.show()
  173.  
  174.  
  175. # In[287]:
  176.  
  177. # Let's replace the imputed values in place of the missing value
  178. X_train_imp_forest = X_train.copy()
  179. X_train_imp_forest[~valid,regressee] = imputed_random_forest[~valid]
  180. # ... and plot the results
  181. plt.figure(figsize=(8,6))
  182. plt.plot(X_train_imp_forest[:,regressee])
  183. plt.show()
  184.  
  185.  
  186. # ### Plotting all the imputations to compare them
  187.  
  188. # In[288]:
  189.  
  190. plt.figure(figsize=(10,10))
  191. plt.plot(X_train[150:251,regressee])
  192. plt.plot(imputed_linear[150:251])
  193. plt.plot(imputed_random_forest[150:251])
  194. plt.legend(["non", "linear", "random forests"])
  195. plt.show()
  196.  
  197.  
  198. # ### Comparing imputation methods
  199. # In order to compare imputation methods, crossvalidation can be used, and their kappa statistic can be observed. Will do this if I have the time.
  200.  
  201. # Training a model
  202. # ===
  203. # Now that Imputation is done, it is possible to train a model to classify the last variable. This has to be a classifier since the last variable is binary.
  204. #
  205. # We are going to use the following
  206. #     1. Naive Bayes: Simple model to prevent overfitting.
  207. #     2. Decision Trees: With post prunning to reduce overfitting.
  208. #     3. Random Forest Classifiers: Make our decision tree model more general.
  209. #
  210. # But first, lets setup the global variables that state the k for k-fold cross validation and the max depth for both of the tree algorithms.
  211.  
  212. # In[289]:
  213.  
  214. cv_folds = 10
  215. tree_max_depth = None
  216.  
  217.  
  218. # ### Naive Bayes
  219. # We will use Gaussian Naive Bayes to classify our data first.
  220. #
  221. # We will check how it performs on:
  222. # * Data without missing values X_train_no_missing
  223. # * Data with mean imputated missing values X_train_imp_mean
  224. # * Data with imputated mising values with linear regression X_train_imp_linear
  225. # * Data with imputated missing values with a Random Forest Regressor X_train_imp_forest
  226.  
  227. # In[293]:
  228.  
  229. from sklearn.naive_bayes import GaussianNB
  230. from sklearn import cross_validation as cv
  231. from sklearn import metrics
  232. clf = GaussianNB()
  233.  
  234. # Remove missing values
  235. bayes_nomissing_pred = cv.cross_val_predict(clf,
  236.                         X_train_no_missing, y_train_no_missing,
  237.                         cv=cv_folds)
  238.  
  239. bayes_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
  240.                                              bayes_nomissing_pred)
  241.  
  242. bayes_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
  243.                                               bayes_nomissing_pred)
  244.  
  245. # Imputation with mean
  246. bayes_mean_pred = cv.cross_val_predict(clf,
  247.                         X_train_imp_mean, y_train,
  248.                         cv=cv_folds)
  249.  
  250. bayes_mean_acc = metrics.accuracy_score(y_train,
  251.                                              bayes_mean_pred)
  252.  
  253. bayes_mean_kap = metrics.cohen_kappa_score(y_train,
  254.                                               bayes_mean_pred)
  255.  
  256. # Imputation with linear regression
  257. bayes_linear_pred = cv.cross_val_predict(clf,
  258.                         X_train_imp_linear, y_train,
  259.                         cv=cv_folds)
  260.  
  261. bayes_linear_acc = metrics.accuracy_score(y_train,
  262.                                              bayes_linear_pred)
  263.  
  264. bayes_linear_kap = metrics.cohen_kappa_score(y_train,
  265.                                               bayes_linear_pred)
  266.  
  267. # Imputation with random forest regressor
  268. bayes_forest_pred = cv.cross_val_predict(clf,
  269.                         X_train_imp_forest, y_train,
  270.                         cv=cv_folds)
  271.  
  272. bayes_forest_acc = metrics.accuracy_score(y_train,
  273.                                              bayes_forest_pred)
  274.  
  275. bayes_forest_kap = metrics.cohen_kappa_score(y_train,
  276.                                               bayes_forest_pred)
  277.  
  278. # No cross validation
  279. clf.fit(X_train_imp_forest, y_train)
  280. bayes_pred = clf.predict(X_train_imp_forest)
  281.  
  282. bayes_acc = metrics.accuracy_score(y_train, bayes_pred)
  283.  
  284. bayes_kap = metrics.cohen_kappa_score(y_train, bayes_pred)
  285. print "\nNo cross validation + Bayes + Impute with Random Forest Regressor"
  286. print "Acc: %.4f Kappa: %.4f" % (bayes_acc,
  287.                                  bayes_kap)
  288. print metrics.confusion_matrix(y_train, bayes_pred)
  289.  
  290. print "\nBayes + Remove missing values"
  291. print "Acc: %.4f Kappa: %.4f" % (bayes_nomissing_acc,
  292.                                  bayes_nomissing_kap)
  293. print metrics.confusion_matrix(y_train_no_missing, bayes_nomissing_pred)
  294.  
  295. print "\nBayes + Impute with Mean"
  296. print "Acc: %.4f Kappa: %.4f" % (bayes_mean_acc,
  297.                                  bayes_mean_kap)
  298. print metrics.confusion_matrix(y_train, bayes_mean_pred)
  299.  
  300. print "\nBayes + Impute with Linear Regressor"
  301. print "Acc: %.4f Kappa: %.4f" % (bayes_linear_acc,
  302.                                  bayes_linear_kap)
  303. print metrics.confusion_matrix(y_train, bayes_linear_pred)
  304.  
  305. print "\nBayes + Impute with Random Forest Regressor"
  306. print "Acc: %.4f Kappa: %.4f" % (bayes_forest_acc,
  307.                                  bayes_forest_kap)
  308. print metrics.confusion_matrix(y_train, bayes_forest_pred)
  309.  
  310.  
  311. # ### Decision Tree
  312.  
  313. # In[294]:
  314.  
  315. from sklearn.tree import DecisionTreeClassifier
  316. clf = DecisionTreeClassifier(random_state=0, max_depth=tree_max_depth)
  317.  
  318. # Remove missing values
  319. tree_nomissing_pred = cv.cross_val_predict(clf,
  320.                         X_train_no_missing, y_train_no_missing,
  321.                         cv=cv_folds)
  322.  
  323. tree_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
  324.                                              tree_nomissing_pred)
  325.  
  326. tree_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
  327.                                               tree_nomissing_pred)
  328.  
  329. # Imputation with mean
  330. tree_mean_pred = cv.cross_val_predict(clf,
  331.                         X_train_imp_mean, y_train,
  332.                         cv=cv_folds)
  333.  
  334. tree_mean_acc = metrics.accuracy_score(y_train,
  335.                                              tree_mean_pred)
  336.  
  337. tree_mean_kap = metrics.cohen_kappa_score(y_train,
  338.                                               tree_mean_pred)
  339.  
  340. # Imputation with linear regression
  341. tree_linear_pred = cv.cross_val_predict(clf,
  342.                         X_train_imp_linear, y_train,
  343.                         cv=cv_folds)
  344.  
  345. tree_linear_acc = metrics.accuracy_score(y_train,
  346.                                              tree_linear_pred)
  347.  
  348. tree_linear_kap = metrics.cohen_kappa_score(y_train,
  349.                                               tree_linear_pred)
  350.  
  351. # Imputation with random forest regressor
  352. tree_forest_pred = cv.cross_val_predict(clf,
  353.                         X_train_imp_forest, y_train,
  354.                         cv=cv_folds)
  355.  
  356. tree_forest_acc = metrics.accuracy_score(y_train,
  357.                                              tree_forest_pred)
  358.  
  359. tree_forest_kap = metrics.cohen_kappa_score(y_train,
  360.                                               tree_forest_pred)
  361.  
  362. # No cross validation
  363. clf.fit(X_train_imp_forest, y_train)
  364. tree_pred = clf.predict(X_train_imp_forest)
  365.  
  366. tree_acc = metrics.accuracy_score(y_train, tree_pred)
  367.  
  368. tree_kap = metrics.cohen_kappa_score(y_train, tree_pred)
  369. print "\nNo cross validation + Decision Tree + Impute with Random Forest Regressor"
  370. print "Acc: %.4f Kappa: %.4f" % (tree_acc,
  371.                                  tree_kap)
  372. print metrics.confusion_matrix(y_train, tree_pred)
  373.  
  374. print "\nDecision Tree + Remove missing values"
  375. print "Acc: %.4f Kappa: %.4f" % (tree_nomissing_acc,
  376.                                  tree_nomissing_kap)
  377. print metrics.confusion_matrix(y_train_no_missing, tree_nomissing_pred)
  378.  
  379. print "\nDecision Tree + Impute with Mean"
  380. print "Acc: %.4f Kappa: %.4f" % (tree_mean_acc,
  381.                                  tree_mean_kap)
  382. print metrics.confusion_matrix(y_train, tree_mean_pred)
  383.  
  384. print "\nDecision Tree + Impute with Linear Regressor"
  385. print "Acc: %.4f Kappa: %.4f" % (tree_linear_acc,
  386.                                  tree_linear_kap)
  387. print metrics.confusion_matrix(y_train, tree_linear_pred)
  388.  
  389. print "\nDecision Tree + Impute with Random Forest Regressor"
  390. print "Acc: %.4f Kappa: %.4f" % (tree_forest_acc,
  391.                                  tree_forest_kap)
  392. print metrics.confusion_matrix(y_train, tree_forest_pred)
  393.  
  394.  
  395. # ### Random Forest Classifier
  396.  
  397. # In[292]:
  398.  
  399. from sklearn.ensemble import RandomForestClassifier
  400. clf = RandomForestClassifier(n_estimators=100,
  401.                              random_state=0,
  402.                              max_depth=tree_max_depth)
  403.  
  404. # Remove missing values
  405. forest_nomissing_pred = cv.cross_val_predict(clf,
  406.                         X_train_no_missing, y_train_no_missing,
  407.                         cv=cv_folds)
  408.  
  409. forest_nomissing_acc = metrics.accuracy_score(y_train_no_missing,
  410.                                              forest_nomissing_pred)
  411.  
  412. forest_nomissing_kap = metrics.cohen_kappa_score(y_train_no_missing,
  413.                                               forest_nomissing_pred)
  414.  
  415. # Imputation with mean
  416. forest_mean_pred = cv.cross_val_predict(clf,
  417.                         X_train_imp_mean, y_train,
  418.                         cv=cv_folds)
  419.  
  420. forest_mean_acc = metrics.accuracy_score(y_train,
  421.                                              forest_mean_pred)
  422.  
  423. forest_mean_kap = metrics.cohen_kappa_score(y_train,
  424.                                               forest_mean_pred)
  425.  
  426. # Imputation with linear regression
  427. forest_linear_pred = cv.cross_val_predict(clf,
  428.                         X_train_imp_linear, y_train,
  429.                         cv=cv_folds)
  430.  
  431. forest_linear_acc = metrics.accuracy_score(y_train,
  432.                                              forest_linear_pred)
  433.  
  434. forest_linear_kap = metrics.cohen_kappa_score(y_train,
  435.                                               forest_linear_pred)
  436.  
  437. # Imputation with random forest regressor
  438. forest_forest_pred = cv.cross_val_predict(clf,
  439.                         X_train_imp_forest, y_train,
  440.                         cv=cv_folds)
  441.  
  442. forest_forest_acc = metrics.accuracy_score(y_train,
  443.                                              forest_forest_pred)
  444.  
  445. forest_forest_kap = metrics.cohen_kappa_score(y_train,
  446.                                               forest_forest_pred)
  447.  
  448. # No cross validation
  449. clf.fit(X_train_imp_forest, y_train)
  450. forest_pred = clf.predict(X_train_imp_forest)
  451.  
  452. forest_acc = metrics.accuracy_score(y_train, forest_pred)
  453.  
  454. forest_kap = metrics.cohen_kappa_score(y_train, forest_pred)
  455. print "\nNo cross validation + Random Forest Classifier + Impute with Random Forest Regressor"
  456. print "Acc: %.4f Kappa: %.4f" % (forest_acc,
  457.                                  forest_kap)
  458. print metrics.confusion_matrix(y_train, forest_pred)
  459.  
  460. print "\nRandom Forest Classifier + Remove missing values"
  461. print "Acc: %.4f Kappa: %.4f" % (forest_nomissing_acc,
  462.                                  forest_nomissing_kap)
  463. print metrics.confusion_matrix(y_train_no_missing,
  464.                                forest_nomissing_pred)
  465.  
  466. print "\nRandom Forest Classifier + Impute with Mean"
  467. print "Acc: %.4f Kappa: %.4f" % (forest_mean_acc,
  468.                                  forest_mean_kap)
  469. print metrics.confusion_matrix(y_train, forest_mean_pred)
  470.  
  471. print "\nRandom Forest Classifier + Impute with Linear Regressor"
  472. print "Acc: %.4f Kappa: %.4f" % (forest_linear_acc,
  473.                                  forest_linear_kap)
  474. print metrics.confusion_matrix(y_train, forest_linear_pred)
  475.  
  476. print "\nRandom Forest Classifier + Impute with Random Forest Regressor"
  477. print "Acc: %.4f Kappa: %.4f" % (forest_forest_acc,
  478.                                  forest_forest_kap)
  479. print metrics.confusion_matrix(y_train, forest_forest_pred)
Add Comment
Please, Sign In to add comment