clear ; close all; clc

%% Setup the parameters you will use for this exercise
input_layer_size  = 400;
hidden_layer_size = 25;
num_labels = 10;
% (note that we have mapped "0" to label 10)

%%  Part 1: Loading and Visualizing Data

fprintf('Loading and Visualizing Data ...\n')
load('ex4data1.mat');
m = size(X, 1);

% Randomly select 100 data points to display
sel = randperm(size(X, 1));
sel = sel(1:100);
displayData(X(sel, :));
fprintf('Program paused. Press enter to continue.\n');
pause;


%% Part 2: Loading Parameters

fprintf('\nLoading Saved Neural Network Parameters ...\n')
load('ex4weights.mat');
nn_params = [Theta1(:) ; Theta2(:)];

%% Part 3: Compute Cost (Feedforward)
fprintf('\nFeedforward Using Neural Network ...\n')

% regularization parameter = 0
lambda = 0;
J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, ...
                   num_labels, X, y, lambda);

fprintf(['Cost at parameters (loaded from ex4weights): %f '...
         '\n'], J);

fprintf('\nProgram paused. Press enter to continue.\n');
pause;

%% Part 4: Implement Regularization

fprintf('\nChecking Cost Function (w/ Regularization) ... \n')

% Weight regularization parameter (we set this to 1 here).
lambda = 1;

J = nnCostFunction(nn_params, input_layer_size, hidden_layer_size, ...
                   num_labels, X, y, lambda);

fprintf(['Cost at parameters (loaded from ex4weights): %f '...
         '\n'], J);

fprintf('Program paused. Press enter to continue.\n');
pause;


%% Part 5: Sigmoid Gradient

fprintf('\nEvaluating sigmoid gradient...\n')

g = sigmoidGradient([-1 -0.5 0 0.5 1]);
fprintf('Sigmoid gradient evaluated at [-1 -0.5 0 0.5 1]:\n  ');
fprintf('%f ', g);
fprintf('\n\n');

fprintf('Program paused. Press enter to continue.\n');
pause;


%% Part 6: Initializing Pameters

fprintf('\nInitializing Neural Network Parameters ...\n')

initial_Theta1 = randInitializeWeights(input_layer_size, hidden_layer_size);
initial_Theta2 = randInitializeWeights(hidden_layer_size, num_labels);
initial_nn_params = [initial_Theta1(:) ; initial_Theta2(:)];


%% Part 7: Implement Backpropagation
fprintf('\nChecking Backpropagation... \n');
checkNNGradients;

fprintf('\nProgram paused. Press enter to continue.\n');
pause;


%% Part 8: Implement Regularization
fprintf('\nChecking Backpropagation (w/ Regularization) ... \n')
lambda = 3;
checkNNGradients(lambda);
debug_J  = nnCostFunction(nn_params, input_layer_size, ...
                          hidden_layer_size, num_labels, X, y, lambda);

fprintf(['\n\nCost at (fixed) debugging parameters (w/ lambda = %f): %f ' ...
         '\n(for lambda = 3, this value should be about 0.576051)\n\n'], lambda, debug_J);
fprintf('Program paused. Press enter to continue.\n');
pause;


%% Part 8: Training NN
fprintf('\nTraining Neural Network... \n')
options = optimset('MaxIter', 50);
lambda = 1;
costFunction = @(p) nnCostFunction(p, ...
                                   input_layer_size, ...
                                   hidden_layer_size, ...
                                   num_labels, X, y, lambda);

[nn_params, cost] = fmincg(costFunction, initial_nn_params, options);
Theta1 = reshape(nn_params(1:hidden_layer_size * (input_layer_size + 1)), ...
                 hidden_layer_size, (input_layer_size + 1));
Theta2 = reshape(nn_params((1 + (hidden_layer_size * (input_layer_size + 1))):end), ...
                 num_labels, (hidden_layer_size + 1));

fprintf('Program paused. Press enter to continue.\n');
pause;


%% Part 9: Visualize Weights

fprintf('\nVisualizing Neural Network... \n')

displayData(Theta1(:, 2:end));

fprintf('\nProgram paused. Press enter to continue.\n');
pause;

%% Part 10: Implement Predict

pred = predict(Theta1, Theta2, X);

fprintf('\nTraining Set Accuracy: %f\n', mean(double(pred == y)) * 100);