In Progress 2

'''
This neural network framework nudges both weights and biases when it performs backpropagation. It can handle multiple outputs. Working on implementing the relu activation function in backpropagation
'''

import numpy as np
import random

class Layer:
    def __init__(self, inputNodes, outputNodes):
        self.weights = 0.1 * np.random.randn(inputNodes, outputNodes)
        self.biases = 1 + np.zeros((1, outputNodes))

    def forward(self, inputs):
        self.output = np.dot(inputs, self.weights) + self.biases
class Activation_ReLU:
    def forward(self, inputs):
        self.output = np.maximum(0, inputs)


learningRate = 0.0000000001
def backwards(network, input_, desired):
    currentLayer = len(network) - 1

    dError = 2*(network[currentLayer].output[0] - desired)

    gradients = np.zeros((len(network), 10)) #The digit here represent maximum number of neurons per layer

    #This assumes the last layer is a normal one and not an activation
    for neuronsPerLastLayer in range(len(network[currentLayer].output[0])):
        gradients[currentLayer][neuronsPerLastLayer] = dError[neuronsPerLastLayer]

    # Start backpropagation for the rest of the layer
    while currentLayer > 0: # Per layer except last one that's connected to the network input

        #Nudge the weights and biases
        for neuronCurrentLayer in range(len(network[currentLayer].output[0])): # Per neuron in current layer
            network[currentLayer].biases[0][neuronCurrentLayer] -= 1 * gradients[currentLayer][neuronCurrentLayer] * learningRate
            for neuronPreviousLayer in range(len(network[currentLayer - 1].output[0])): # Per neuron in previous layer/per weight per neuron in current layer
                network[currentLayer].weights[neuronPreviousLayer][neuronCurrentLayer] -= network[currentLayer - 1].output[0][neuronPreviousLayer] * gradients[currentLayer][neuronCurrentLayer] * learningRate

        # Calculate gradients for every neuron in the next layer you're going to adjust
        if type(network[currentLayer - 1]) == Activation_ReLU:
            for neuronCurrentLayer in range(len(network[currentLayer].output[0])): # Per neuron in current layer
                for neuronPreviousLayer in range(len(network[currentLayer - 2].output[0])): # Per neuron in previous normal layer (skips activation layer)
                    if(network[currentLayer - 2].output[0][neuronPreviousLayer] > 0):
                        gradients[currentLayer - 2][neuronPreviousLayer] += network[currentLayer].weights[neuronPreviousLayer][neuronCurrentLayer] * gradients[currentLayer][neuronCurrentLayer]
                    else:
                        gradients[currentLayer - 2][neuronPreviousLayer] = 0

            currentLayer -= 2
        else:
            for neuronCurrentLayer in range(len(network[currentLayer].output[0])): # Per neuron in current layer
                for neuronPreviousLayer in range(len(network[currentLayer - 1].output[0])): # Per neuron in previous layer
                    gradients[currentLayer - 1][neuronPreviousLayer] += network[currentLayer].weights[neuronPreviousLayer][neuronCurrentLayer] * gradients[currentLayer][neuronCurrentLayer]

            currentLayer -= 1

    #print("Error: ", (network[len(network) - 1].output[0] - desired))
    #error = network[len(network) - 1].output[0] - desired
    #print("Gradients total: \n", gradients)





#Create training data
#inputs = [3, 6, 2, 8, 12, 90, 45, 23, 88, 18]
#desired = np.array([[6, 6], [12, 12], [4, 4], [16, 16], [24, 24], [180, 180], [90, 90], [46, 46], [176, 176], [36, 36]])
#inputs = [4, 6, 1, 3, 9, 2, 3, 7, 10, 34]
#desired = [8, 12, 2, 6, 18, 4, 6, 14, 20, 68]

inputs = []
desired = []

for y in range(1000):
    inputs.append(y + 1)

random.shuffle(inputs)

for y in range(1000):
    desired.append(inputs[y] * 2)


#Create neural network
layer1 = Layer(1, 3)
'''
Layer1 weights aren't going to be affected by the backwards function, so juts set the weights to 1 if you want like this:
layer1.weights *= 1/layer1.weights
'''
layer1.weights *= 1/layer1.weights

layer2 = Layer(3, 1)
activation2 = Activation_ReLU()

layer3 = Layer(3, 3)
activation3 = Activation_ReLU()

layer4 = Layer(3, 1)

#Train the network
for samples_in_batch in range(100):
    for x in range(len(inputs)):

        #With activations
        '''
        layer1.forward(inputs[x])
        layer2.forward(layer1.output)
        activation2.forward(layer2.output)
        layer3.forward(activation2.output)
        activation3.forward(layer3.output)
        layer4.forward(activation3.output)

        backwards([layer1, layer2, activation2, layer3, activation3, layer4], inputs[x], desired[x])
        '''

        #Without activations
        layer1.forward(inputs[x])
        layer2.forward(layer1.output)
        '''
        layer3.forward(layer2.output)
        layer4.forward(layer3.output)
        '''

        backwards([layer1, layer2], inputs[x], desired[x])


#Test the network
testInput = 24

#With activations
'''
layer1.forward(testInput)
layer2.forward(layer1.output)
activation2.forward(layer2.output)
layer3.forward(activation2.output)
activation3.forward(layer3.output)
layer4.forward(activation3.output)
#backwards([layer1, layer2, activation2, layer3, activation3, layer4], testInput, 48)

'''

#Without activations
layer1.forward(testInput)
layer2.forward(layer1.output)
'''
layer3.forward(layer2.output)
layer4.forward(layer3.output)
'''
#backwards([layer1, layer2, activation2, layer3, activation3, layer4], testInput, 48)


print("Guess: ", layer2.output)
print("Error: ", layer2.output - (testInput * 2))