############################### Training setup ##################################

#Define 10 folds:
seed = 7
np.random.seed(seed)
kfold = KFold(n_splits=10, shuffle=False, random_state=None)
print "Splits"
cvscores_acc = []
cvscores_loss = []
hist = []
i = 0
#train_set_data_vstacked_normalized_reshaped = np.reshape(train_set_data_vstacked_normalized,train_set_data_vstacked_normalized.shape+(1,))
#train_set_output_vstacked_normalized_reshaped = np.reshape(train_set_output_vstacked_normalized,train_set_output_vstacked_normalized.shape+(1,))
for train, test in kfold.split(train_set_data_vstacked_normalized):

    print "Model definition!"
    model = Sequential()

    #act = PReLU(init='normal', weights=None)
    #model.add(Dense(output_dim=400,input_dim=400, init="normal", activation=K.tanh))

    #act1 = PReLU(init='normal', weights=None)

    #act2 = PReLU(init='normal', weights=None)
    model.add(Dense(output_dim=2050, input_dim=2050, init="normal",activation='tanh'))

    model.add(Dense(output_dim=13, input_dim=2050, init="normal",activation='tanh'))
    model.add(Lambda(lambda x: numpy_unorm(x)))

    #model.add(ELU(100))

    #model.add(Convolution1D(13, 3, border_mode='same', input_shape=(2050,1)))

    print "Compiling"
    #rms_opt = keras.optimizers.RMSprop(lr=0.01, rho=0.9, epsilon=1e-08, decay=0.0)
    model.compile(loss='mean_squared_error', optimizer="RMSprop")
    print "Compile done! "

    print '\n'

    print "Train start"

    reduce_lr=ReduceLROnPlateau(monitor='val_loss', factor=0.01, patience=3, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0.00000001)
    stop  = EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=1, mode='auto')

    log=csv_logger = CSVLogger('training_'+str(i)+'.csv')


    hist_current = model.fit(train_set_data_vstacked_normalized[train],
                                train_set_output_vstacked[train],
                                shuffle=False,
                                validation_data=(train_set_data_vstacked_normalized[test],train_set_output_vstacked[test]),
                                validation_split=0.1,
                                nb_epoch=150,
                                verbose=1,
                                callbacks=[reduce_lr,log,stop])

    hist.append(hist_current)

    print()
    print model.summary()
    print "Model stored"
    model.save("Model"+str(i)+".h5")
    model.save_weights("Model"+str(i)+"_weights.h5")
    del model
    # serialize model to YAML
    #model_yaml = model.to_yaml()
    #with open("model.yaml", "w") as yaml_file:
    #    yaml_file.write(model_yaml)
        # serialize weights to HDF5
    #    model.save_weights("Model"+str(i)+".h5")
    #    print("Saved model to disk")

    print "New Model:"
    i=i+1

    #print("%.2f%% (+/- %.2f%%)" % (numpy.mean(cvscores_acc), numpy.std(cvscores_acc)))
    #print("%.2f%% (+/- %.2f%%)" % (numpy.mean(cvscores_loss), numpy.std(cvscores_loss)))