<Type1>540_test Data Preprocessing

import pandas as pd
import numpy as np
import h5py
import matplotlib
import matplotlib.pyplot as plt
from keras.models import Model, Sequential
from keras.layers import Dense, Input, concatenate,Conv1D, Conv2D, MaxPooling2D, Conv2DTranspose,MaxPooling1D, Cropping2D, Multiply, subtract, Flatten, Reshape, Permute, LSTM, TimeDistributed,Dropout,BatchNormalization,UpSampling1D
from keras.optimizers import SGD
from keras.callbacks import ModelCheckpoint,EarlyStopping
import tensorflow as tf
from keras.models import load_model
from keras import optimizers
from keras import regularizers
from math import sqrt
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
import pickle
from sklearn.preprocessing import StandardScaler
import time
from sklearn import preprocessing
from numpy import argmax
from keras.utils import to_categorical
from tabulate import tabulate
from numpy import array



#implement the data
file_540=open("D:\\NCSU Research\\GEARS_Jul9\\GEARS\\540\\x_test_540.pkl","rb")
data = pickle.load(file_540)

##判断是否有缺失值
#print(data.isnull().sum()) 有且某几个feature超过一半



##统计缺失值情况
def collect_nan_val(dataframe):
    return dataframe.isna().sum() / dataframe.shape[0]*100

print(collect_nan_val(data))


data=data.fillna(data.median(),inplace=False)
#print(data_complete)

data["label"]=pd.cut(data["glucose level"], bins=[0,70,130,10000], labels=[-1,0,1])

data_without=data.drop(labels=["time","glucose level"],axis=1,inplace=False)

data_0=data_without
data_0=data_0.reset_index(drop=True)


data_1=data_without.drop(data.index[0])
data_1.columns=["fingerstick_1", "basis_gsr_1", "basis_skintem_1", "acceleration_1", "bolus_1", "basal_1", "meal_1", "basis_sleep_1", "label_1"]
data_1=data_1.reset_index(drop=True) #after reseting will give a new dataframe
#print(data_1)
data_2=data_1.drop(data_1.index[0])
data_2.columns=["fingerstick_2", "basis_gsr_2", "basis_skintem_2", "acceleration_2", "bolus_2", "basal_2", "meal_2", "basis_sleep_2", "label_2"]
data_2=data_2.reset_index(drop=True)

data_3=data_2.drop(data_2.index[0])
data_3.columns=["fingerstick_3", "basis_gsr_3", "basis_skintem_3", "acceleration_3", "bolus_3", "basal_3", "meal_3", "basis_sleep_3", "label_3"]
data_3=data_3.reset_index(drop=True)

data_4=data_3.drop(data_3.index[0])
data_4.columns=["fingerstick_4", "basis_gsr_4", "basis_skintem_4", "acceleration_4", "bolus_4", "basal_4", "meal_4", "basis_sleep_4", "label_4"]
data_4=data_4.reset_index(drop=True)

data_5=data_4.drop(data_4.index[0])
data_5.columns=["fingerstick_5", "basis_gsr_5", "basis_skintem_5", "acceleration_5", "bolus_5", "basal_5", "meal_5", "basis_sleep_5", "label_5"]
data_5=data_5.reset_index(drop=True)

data_6=data_5.drop(data_5.index[0])#though index is always 0, the output for data_6 is began at 6
data_6.columns=["fingerstick_6", "basis_gsr_6", "basis_skintem_6", "acceleration_6", "bolus_6", "basal_6", "meal_6", "basis_sleep_6", "label_6"]
data_6=data_6.reset_index(drop=True)


data_new=pd.concat([data_0,data_1,data_2,data_3,data_4,data_5,data_6],axis=1)
#print(data_new.dropna())
data_new=data_new.dropna()#cut the line that include "NAN"


#get the new label for the new data set
i=0
len_label=len(data_new)
label_new=[]
searching_list=data_6["label_6"]
#print(type(searching_list))
searching_list=searching_list.tolist()
print(len(searching_list)) #type: list len:13103
#for i in range(3):
#    print(type(int((data.loc[str(i)]["label"]))))==int
#print(len_label)=13103




while i<=len(searching_list)-7:#len_label
    j=0
    label_list=[]
    g=i
    w=0
    for j in range(7): #[0,6]
        k=searching_list[g]
        label_list.append(int(k))
        g=g+1
    #print(label_list)
    for m in label_list:#未考虑-1,1在30分钟内同时出现的情况！！！
        if m>0:
            w=w+1
        elif m==0:
            w=w
        else:
            w=w-1
    #print(w)
    if w>0:
        label_new.append(1)
    elif w==0:
        label_new.append(0)
    else:
        label_new.append(-1)
    i=i+1
#print(len(label_new))#=13097

#change the lbael list into dataframe and combine with the data_new
from pandas.core.frame import DataFrame
n={"label_new": label_new}
data_label=DataFrame(n)
data_complete=pd.concat([data_new,data_label],axis=1)
data_complete=data_complete.dropna()#cut the line that include "NAN"

#count label
#list_c=data_complete["label"].values.tolist()
#print(dict(zip(*np.unique(list_c, return_counts=True)))) {-1: 142, 0: 1083, 1: 1835}

data_complete.to_csv("D:\\NCSU Research\\GEARS_Jul9\\GEARS\\540\\data_test.complete.csv")