# Create KNN
knn_model = KNeighborsClassifier()

# Number of neighbors
n_neighbors = range(3, 5, 10)

# Create regularization term space
weights = ["uniform", "distance"]

# Create distance metrics space
dist_metrics = [1, 2]

# Create hyperparameters
hyperparameters = dict(n_neighbors = n_neighbors, weights = weights, p = dist_metrics)

# Create GridSearch
knn_gs = GridSearchCV(knn_model, hyperparameters, cv = 5, scoring = "f1_micro")

# Fit GridSearch
model_knn = knn_gs.fit(X_train, y_train)

# Model best estimators
print("Number of Neighbors: ", model_knn.best_estimator_.get_params()["n_neighbors"])
print("Weighting Type: ", model_knn.best_estimator_.get_params()["weights"])
print("Distance Metric: ", model_knn.best_estimator_.get_params()["p"])

# Predictions
y_train_pred_knn = model_knn.best_estimator_.predict(X_train)
y_test_pred_knn = model_knn.best_estimator_.predict(X_test)

# Print F1 scores
print("Training F1 Micro Average: ", f1_score(y_train, y_train_pred_knn, average = "micro"))
print("Test F1 Micro Average: ", f1_score(y_test, y_test_pred_knn, average = "micro"))

# Confusion Matrix
knn_cm = confusion_matrix(y_test, y_test_pred_knn)
print(knn_cm)

plt.figure(figsize = (9, 9))
sns.heatmap(knn_cm, annot = True, fmt = ".3f", linewidths = .5, square = True, cmap = "BuPu")

# Classification Report
print(classification_report(y_test, y_test_pred_knn))