left_branch = Sequential()
left_branch.add(Dense(32, input_dim=X_train.shape[1]))
left_branch.add(Embedding(input_dim=vocab_size+1, output_dim=2, input_length=4))
left_branch.add(Flatten())

right_branch = Sequential()
right_branch.add(Dense(32, input_dim=X_train_tfidf.shape[1]))
merged = Merge([left_branch, right_branch], mode='concat')

model = Sequential()
model.add(merged)
model.add(Dense(32, activation='tanh'))

#block 1
model.add(Dense(units=64, activation='tanh', name='first'))
#model.add(Dropout(0.005))
model.add(BatchNormalization())
block 2
model.add(Dense(units=64, activation='relu', name='second'))
#model.add(Dropout(0.005))
model.add(BatchNormalization())
#block 3
model.add(Dense(units=128, activation='relu', name='third'))
#model.add(Dropout(0.005))
model.add(BatchNormalization())
#block 4
model.add(Dense(units=128, activation='relu', name='fourth'))
#model.add(Dropout(0.005))
model.add(BatchNormalization())
#block 5
model.add(Dense(units=256, activation='relu', name='fifth'))
#model.add(Dropout(0.005))
model.add(BatchNormalization())
#block 6
model.add(Dense(units=256, activation='relu', name='sixth'))
#model.add(Dropout(0.005))
model.add(BatchNormalization())
#block 7
model.add(Dense(units=128, activation='relu', name='seventh'))
#model.add(Dropout(0.005))
model.add(BatchNormalization())
block 8
model.add(Dense(units=64, activation='tanh', name='eighth'))
#model.add(Dropout(0.005))
model.add(BatchNormalization())
model.add(Dense(1, activation='sigmoid')) #sigmoid
sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
model.summary()
#final_model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy']) #sgd3 #loss='mse'
model.fit([X_train, X_train_tfidf], y_train,
         validation_split=0.2, #validation_data=(np.array(X_test.todense()), y_test),
         epochs=5, batch_size=32)