##import packages
import pandas as pd
from sklearn import preprocessing
import sklearn.model_selection as ms
from sklearn import linear_model
import sklearn.metrics as sklm
import numpy as np
import numpy.random as nr
import matplotlib.pyplot as plt
import seaborn as sns
import scipy.stats as ss
import math
%matplotlib inline

##load datasets with pandas
train_values = pd.read_csv('train_values.csv')
train_labels = pd.read_csv('train_labels.csv')
test_values = pd.read_csv('test_values.csv')

##viewing value data type
train_values.dtypes

##view columns with null values and count them
print(train_values.isnull().sum())
train_values.shape

##drop columns with too many missing values
train_values.drop(['bank_interest_rate'], axis = 1, inplace = True)
train_values.drop(['mm_interest_rate'], axis = 1, inplace = True)
train_values.drop(['mfi_interest_rate'], axis = 1, inplace = True)
train_values.drop(['other_fsp_interest_rate'], axis = 1, inplace = True)
train_values.shape

##drop rows with missing values
cols = ['education_level', 'share_hh_income_provided']
for column in cols:
    train_values.loc[train_values[column].isnull()] = np.nan
    train_values.dropna(axis = 0, inplace = True)
train_values.shape

##checking if it worked
print(train_values.isnull().sum())

##merging data into one frame
training_set = pd.merge(train_values, train_labels, on = 'row_id')
training_set.head(1)

##encoding categorical features into binary numeric data
def encode_string(cat_feature):
    enc = preprocessing.LabelEncoder()
    enc.fit(cat_feature)
    enc_cat_feature = enc.transform(cat_feature)
    ohe = preprocessing.OneHotEncoder()
    encoded = ohe.fit(enc_cat_feature.reshape(-1,1))
    return encoded.transform(enc_cat_feature.reshape(-1,1)).toarray()

categorical_columns = ['religion', 'relationship_to_hh_head',
                       'employment_category_last_year', 'employment_type_last_year']
Features = encode_string(training_set['country'])
for col in categorical_columns:
    temp = encode_string(training_set[col])
    Features = np.concatenate([Features, temp], axis = 1)
print(Features.shape)

##adding on the numeric columns
Features = np.concatenate([Features, np.array(training_set[['is_urban', 'age', 'female', 'married',
       'education_level', 'literacy', 'can_add',
       'can_divide', 'can_calc_percents', 'can_calc_compounding',
       'employed_last_year',
       'share_hh_income_provided',
       'income_ag_livestock_last_year', 'income_friends_family_last_year',
       'income_government_last_year', 'income_own_business_last_year',
       'income_private_sector_last_year', 'income_public_sector_last_year',
       'num_times_borrowed_last_year', 'borrowing_recency', 'formal_savings',
       'informal_savings', 'cash_property_savings', 'has_insurance',
       'has_investment', 'num_shocks_last_year',
       'avg_shock_strength_last_year', 'borrowed_for_emergency_last_year',
       'borrowed_for_daily_expenses_last_year',
       'borrowed_for_home_or_biz_last_year', 'phone_technology', 'can_call',
       'can_text', 'can_use_internet', 'can_make_transaction',
       'phone_ownership', 'advanced_phone_use', 'reg_bank_acct', 'reg_mm_acct',
       'reg_formal_nbfi_account', 'financially_included', 'active_bank_user',
       'active_mm_user', 'active_formal_nbfi_user',
       'active_informal_nbfi_user', 'nonreg_active_mm_user',
       'num_formal_institutions_last_year',
       'num_informal_institutions_last_year',
       'num_financial_activities_last_year']])], axis = 1)
Features.shape