#Split the data into training (80%) and testing (20%) sets
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.20,random_state=42)

#Function for hyperparameter optimization
def fitAndScore(logL1):
    #Hyperparamter to be optimized is the log of the l1 penalty term
    l1 = np.exp(logL1)
    #Instatiate and fit the lasso model to the training data
    model = LogisticRegression(C=l1,penalty='l1',solver='liblinear')
    model.fit(X_train,y_train)
    #Use the trained model to predict the probability that y=1 in the test data
    y_pred = model.predict_proba(X_test)[:,1]
    #Objective to maximize is the negative log loss (binary cross entropy) on the testing data
    obj = -log_loss(y_test,y_pred)
    return obj

#Set the bounds for optimizing the l1 penalty parameter
pb = [np.log(1.0E-4),np.log(10000000.)]
pbounds = {'logL1': list(pb)}
#Instatiate a bayesian optimization (basically just optimization with a gaussian process surrogate model)
optimizer = BayesianOptimization(f=fitAndScore,pbounds=pbounds,verbose=2,random_state=1)
#Run the optimizer (this is totally overkill for this problem and it's good enough to just do a grid search for l1 but this is just to show how things work)
optimizer.maximize(init_points=100,n_iter=0)