gs = GridSearchCV(estimator=some_classifier,
                  param_grid=some_grid,
                  cv=5,
                  scoring=make_scorer(custom_scorer))

gs.fit(training_data, training_y)

def custom_scorer(y, y_pred):
"""
(1) y contains ground truths, but only for the left-out fold
(2) Similarly, y_pred contains predicted probabilities, but only for the left-out fold
(3) So y, y_pred is each of length ~len(training_y)/5
"""

return scaler_value