Neural_Network_from_scratch.py

from numpy import dot, random, exp, array, mean, seterr
import math

random.seed(1)

n_classes = 1

#hidden_layer sizes 5 nodes and 4 nodes
n_nodes = [5,4]

training_set_inputs = array([[0,0,1],[1,1,1],[1,0,1],[0,1,1],[1,0,0]])
training_set_outputs = array([[0,1,1,0,1]]).T

weights_biases = []
layers = []

#this code is for implementing biases. This may help with the output but I don't know how to
#backpropgate biases
"""for j in range(len(n_nodes)):
    if j == 0:
        weights_biases.append({"weights": random.randn(len(training_set_inputs[0]), n_nodes[j]),"biases": random.randn(n_nodes[j])})
    else:
        weights_biases.append({"weights": random.randn(n_nodes[j-1], n_nodes[j]),"biases": random.randn(n_nodes[j])})
weights_biases.append({"weights": random.randn(n_nodes[-1], n_classes),"biases": random.randn(n_classes)})

def neural_network(data, g_nodes):
    for j in range(len(g_nodes)):
        if j == 0:
            hidden_layer = np.add(dot(data, weights_biases[j]["weights"]), weights_biases[j]["biases"])
            layers.append(sigmoid(hidden_layer))
        elif j != 0:
            hidden_layer = np.add(dot(layers[j-1], weights_biases[j]["weights"]), weights_biases[j]["biases"])
            layers.append(sigmoid(hidden_layer))

    output = np.add(dot(layers[-1], weights_biases[-1]["weights"]), weights_biases[-1]["biases"])
    return output
"""
#we set the initial weights to random because we do not have any other suggestion
#if what they should be yet
#the biases are there so that if the weights return a value of 0  the bias
#can correct that
for j in range(len(n_nodes)):
    if j == 0:
        weights_biases.append({"weights": random.randn(len(training_set_inputs[0]), n_nodes[j])})
    elif len(n_nodes) > 1 and j != 0:
        weights_biases.append({"weights": random.randn(n_nodes[j-1], n_nodes[j])})
weights_biases.append({"weights": random.randn(n_nodes[-1], n_classes)})

def neural_network(data, g_nodes):
    #this updates the values for the hidden layers by using the weights and
    #the values from the previous hidden layer to update them
    for j in range(len(g_nodes)):
        if j == 0:
            hidden_layer = dot(data, weights_biases[j]["weights"])
            layers.append(sigmoid(hidden_layer))
        elif len(n_nodes) > 1 and j != 0:
            hidden_layer = dot(layers[j-1], weights_biases[j]["weights"])
            layers.append(sigmoid(hidden_layer))

    layers.append(dot(layers[-1], weights_biases[-1]["weights"]))
    return layers[-len(n_nodes) - 1:]

#sigmoid activation function
def sigmoid(x):
  return 1 / (1 + exp(-x))

#derivative of sigmoid
def sigmoid_derivative(x):
    return x * (1 - x)

#training function that trains and updates the weights in our neural net
def train(target, num_iterations, learning_rate):
    for i in range(num_iterations):
        #see what results our neural net gives us now
        layers_vals = neural_network(training_set_inputs, n_nodes)

        #output from the hidden layers
        output = layers_vals[-1]

        #deltas are the change needed to adjust the weights
        delta4 = (target - output) * sigmoid_derivative(output)

        delta3 = dot(weights_biases[2]["weights"], delta4.T)*(sigmoid_derivative(layers_vals[1]).T)
        delta2 = dot(weights_biases[1]["weights"], delta3)*(sigmoid_derivative(layers_vals[0]).T)

        adjustment3 = dot(layers_vals[1].T, delta4) * learning_rate
        adjustment2 = dot(layers_vals[0].T, delta3.T) * learning_rate
        adjustment1 = dot(training_set_inputs.T, delta2.T) * learning_rate

        #update the weights
        weights_biases[0]["weights"] += adjustment1
        weights_biases[1]["weights"] += adjustment2
        weights_biases[2]["weights"] += adjustment3

        if i % 100 == 0:
            print str(num_iterations/100) + " epoch error: " + str(mean(abs(target-output)))
            print output
train(training_set_outputs, 20000, 0.01)