NeuralNetworkFrameworkBetaVersion3

'''
This neural network framework only nudges weights when it performs backpropagation. It can handle multiple outputs.
'''

import numpy as np
import random

class Layer:
    def __init__(self, inputNodes, outputNodes):
        self.weights = 0.1 * np.random.randn(inputNodes, outputNodes)
        self.biases = np.zeros((1, outputNodes))

    def forward(self, inputs):
        self.output = np.dot(inputs, self.weights) + self.biases
class Activation_ReLU:
    def forward(self, inputs):
        self.output = np.maximum(0, inputs)

learningRate = 0.0000001
def backwards(network, input_, desired):
    currentLayer = len(network) - 1


    dError = 2*(network[currentLayer].output[0] - desired)

    gradients = np.zeros((len(network), 5)) #The digit here represent maximum number of neurons per layer

    for neuronsPerLastLayer in range(len(network[currentLayer].output[0])):
        gradients[currentLayer][neuronsPerLastLayer] = dError[neuronsPerLastLayer]

    currentLayer = len(network) - 1
    while currentLayer > 0: # Per layer
        if type(network[currentLayer - 1]) == Activation_ReLU:
            pass
        else:
                #Nudge the weights
                for neuronCurrentLayer in range(len(network[currentLayer].output[0])): # Per neuron in current layer
                    for neuronPreviousLayer in range(len(network[currentLayer - 1].output[0])): # Per neuron in previous layer
                        network[currentLayer].weights[neuronPreviousLayer][neuronCurrentLayer] -= network[currentLayer - 1].output[0][neuronPreviousLayer] * gradients[currentLayer][neuronCurrentLayer] * learningRate


                # Calculate gradients for every neuron in the next layer you're going to adjust
                for neuronCurrentLayer in range(len(network[currentLayer].output[0])): # Per neuron in current layer
                    #print("Neuron ", neuronCurrentLayer + 1, ": ")
                    for neuronPreviousLayer in range(len(network[currentLayer - 1].output[0])): # Per neuron in previous layer
                        gradients[currentLayer - 1][neuronPreviousLayer] += network[currentLayer].weights[neuronPreviousLayer][neuronCurrentLayer] * gradients[currentLayer][neuronCurrentLayer]

        currentLayer -= 1 #Go to previous layer
    #print("Gradients: 2: ", gradients)
    print("Error: ", (network[len(network) - 1].output[0] - desired))

#Create training data
inputs = [3, 6, 2, 8, 12, 90, 45, 23, 88, 18]
desired = np.array([[6, 6], [12, 12], [4, 4], [16, 16], [24, 24], [180, 180], [90, 90], [46, 46], [176, 176], [36, 36]])
#inputs = 12
#desired = np.array([24, 24])

#Create neural network
layer1 = Layer(1, 5)

layer2 = Layer(5, 3)

layer3 = Layer(3, 4)

layer4 = Layer(4, 2)

#Train the network
for iteration in range(5000):
    for x in range(len(inputs)):
        layer1.forward(inputs[x])
        layer2.forward(layer1.output)
        layer3.forward(layer2.output)
        layer4.forward(layer3.output)
        backwards([layer1, layer2, layer3, layer4], inputs[x], desired[x])

#Test the network
userInput = 333
layer1.forward(userInput)
layer2.forward(layer1.output)
layer3.forward(layer2.output)
layer4.forward(layer3.output)

print("Guess: ", layer4.output)