NeuralNetworkFrameworkBetaVersionStrongestAndLatest

'''
This neural network framework nudges both weights and biases in all layers when it performs backpropagation. It can handle multiple outputs and inputs. Working on implementing the relu activation function in backpropagation (try it out it should work)
'''

import numpy as np
import random

class Layer:
    def __init__(self, inputNodes, outputNodes):
        self.weights = 0.1 * np.random.randn(inputNodes, outputNodes)
        self.biases = 0 + np.zeros((1, outputNodes))

    def forward(self, inputs):
        self.output = np.dot(inputs, self.weights) + self.biases
class Activation_ReLU:
    def forward(self, inputs):
        self.output = np.maximum(0, inputs)


learningRate = 0.00000001
def backwards(network, input_, desired):
    currentLayer = len(network) - 1

    dError = 2*(network[currentLayer].output[0] - desired)

    gradients = np.zeros((len(network), 10)) #The digit here represent maximum number of neurons per layer

    #This presumes the last layer is a normal one and not an activation
    for neuronsPerLastLayer in range(len(network[currentLayer].output[0])):
        gradients[currentLayer][neuronsPerLastLayer] = dError[neuronsPerLastLayer]

    # Start backpropagation for the rest of the layer
    while currentLayer >= 0: # Per layer except last one that's connected to the network input
        if currentLayer == 0:
            # Nudge the weights and biases in the first layer
            for neuronCurrentLayer in range(len(network[currentLayer].output[0])): # Per neuron in current layer
                network[currentLayer].biases[0][neuronCurrentLayer] -= 1 * gradients[currentLayer][neuronCurrentLayer] * learningRate
                for neuronPreviousLayer in range(len(input_)): # Per neuron in previous layer
                    network[currentLayer].weights[neuronPreviousLayer][neuronCurrentLayer] -= input_[neuronPreviousLayer] * gradients[currentLayer][neuronCurrentLayer] * learningRate

            currentLayer -= 1
        else:
            #Nudge the weights and biases
            for neuronCurrentLayer in range(len(network[currentLayer].output[0])): # Per neuron in current layer
                network[currentLayer].biases[0][neuronCurrentLayer] -= 1 * gradients[currentLayer][neuronCurrentLayer] * learningRate
                for neuronPreviousLayer in range(len(network[currentLayer - 1].output[0])): # Per neuron in previous layer/per weight per neuron in current layer
                    network[currentLayer].weights[neuronPreviousLayer][neuronCurrentLayer] -= network[currentLayer - 1].output[0][neuronPreviousLayer] * gradients[currentLayer][neuronCurrentLayer] * learningRate

            # Calculate gradients for every neuron in the next layer you're going to adjust
            if type(network[currentLayer - 1]) == Activation_ReLU:
                for neuronCurrentLayer in range(len(network[currentLayer].output[0])): # Per neuron in current layer
                    for neuronPreviousLayer in range(len(network[currentLayer - 2].output[0])): # Per neuron in previous normal layer (skips activation layer)
                        if(network[currentLayer - 2].output[0][neuronPreviousLayer] > 0):
                            gradients[currentLayer - 2][neuronPreviousLayer] += network[currentLayer].weights[neuronPreviousLayer][neuronCurrentLayer] * gradients[currentLayer][neuronCurrentLayer]
                        else:
                            gradients[currentLayer - 2][neuronPreviousLayer] = 0

                currentLayer -= 2
            else:
                for neuronCurrentLayer in range(len(network[currentLayer].output[0])): # Per neuron in current layer
                    for neuronPreviousLayer in range(len(network[currentLayer - 1].output[0])): # Per neuron in previous layer
                        gradients[currentLayer - 1][neuronPreviousLayer] += network[currentLayer].weights[neuronPreviousLayer][neuronCurrentLayer] * gradients[currentLayer][neuronCurrentLayer]

                currentLayer -= 1

    #print("Error: ", (network[len(network) - 1].output[0] - desired))
    #error = network[len(network) - 1].output[0] - desired
    #print("Gradients total: \n", gradients)


#Create training data
#inputs = [3, 6, 2, 8, 12, 90, 45, 23, 88, 18]
#desired = np.array([[6, 6], [12, 12], [4, 4], [16, 16], [24, 24], [180, 180], [90, 90], [46, 46], [176, 176], [36, 36]])
#inputs = [4, 6, 1, 3, 9, 2, 3, 7, 10, 34]
#desired = [8, 12, 2, 6, 18, 4, 6, 14, 20, 68]

inputs = []
desired = []

for y in range(1000):
    inputs.append(y + 1)

random.shuffle(inputs)

for y in range(1000):
    desired.append(inputs[y] * 2)


#Create neural network
layer1 = Layer(1, 3)

layer2 = Layer(3, 3)
activation2 = Activation_ReLU()

layer3 = Layer(3, 3)
activation3 = Activation_ReLU()

layer4 = Layer(3, 1)

#Train the network
for samples_in_batch in range(10):
    for x in range(len(inputs)):

        '''
        #With activations
        layer1.forward(inputs[x])
        layer2.forward(layer1.output)
        activation2.forward(layer2.output)
        layer3.forward(activation2.output)
        activation3.forward(layer3.output)
        layer4.forward(activation3.output)

        backwards([layer1, layer2, activation2, layer3, activation3, layer4], inputs[x], desired[x])
        '''

        #Without activations
        layer1.forward(inputs[x])
        layer2.forward(layer1.output)

        layer3.forward(layer2.output)
        layer4.forward(layer3.output)

        #inputToList = [inputs[x]]
        backwards([layer1, layer2, layer3, layer4], [inputs[x]], desired[x])

#Test the network
testInput = 333 #Always has to be a list or np.array even if it's just one input element

#With activations
'''
layer1.forward(testInput)
layer2.forward(layer1.output)
activation2.forward(layer2.output)
layer3.forward(activation2.output)
activation3.forward(layer3.output)
layer4.forward(activation3.output)
#backwards([layer1, layer2, activation2, layer3, activation3, layer4], testInput, 48)
'''

#Without activations
layer1.forward(testInput)
layer2.forward(layer1.output)
layer3.forward(layer2.output)
layer4.forward(layer3.output)

backwards([layer1, layer2, layer3, layer4], [testInput], testInput*2)

print("Guess: ", layer4.output)
print("Error: ", layer4.output - (testInput * 2))