# Train and test over multiple train/validation sets
early_stop = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, patience=5, verbose=2, mode='auto', restore_best_weights=False)
sss2 = StratifiedShuffleSplit(n_splits=10, test_size=0.2, random_state=1).split(train_x, train_labels)
for i in range(10):
    train_indices_2, val_indices = next(sss2)
    model = getModel()
    model.fit(x=train_x[train_indices_2], y=train_labels[train_indices_2], epochs=50, batch_size=32, shuffle=True, validation_data = (train_x[val_indices], train_labels[val_indices]), verbose=2, callbacks=[early_stop])#Line7
    test_loss, test_accuracy = model.evaluate(test_x, test_labels, verbose=2)
    print (test_loss, test_accuracy)
    predicted = model.predict(test_x, verbose=2)
    predicted_labels = predicted.argmax(axis=1)
    print (confusion_matrix(labels[test_indices], predicted_labels))
    print (classification_report(labels[test_indices], predicted_labels, digits=4, target_names=namesInLabelOrder))