for j in range(iterations):
    #reset loss
    totalLoss = 0
    print("\niteration ",j+1)
    for i in range(0,X_train.shape[0]):

        #predict output for one image
        predicted = Pred(X_train[i],W1,b1,W2,b2)

        #add loss
        totalLoss += loss(predicted,y_train[i])
        #predict loss given the previous losses, in order to display it before having completed the iteration
        predictedLoss = totalLoss * X_train.shape[0] / i

        #Update status every few times
        if i%(X_train.shape[0]/20) == 0 :
            print(int(i/X_train.shape[0]*100),"%            |loss:",predictedLoss,end='\r')

        #get the weight derivatives
        dweight2 = dW2(W2,W1,b1,X_train[i],predicted,y_train[i])
        dweight1 = dW1(W2,W1,b1,X_train[i],predicted,y_train[i])

        #update all model parameters using the given learning rate
        W2 -= lr*dweight2
        W1 -= lr*dweight1
        b1 -= lr* db1(W2,W1,b1,X_train[i],predicted,y_train[i])
        b2 -= lr* db2(predicted,y_train[i])

    #after each iteration, add loss to plot
    lossPlot.append(-totalLoss)
    #dynamic plot during training
    ax.clear()
    ax.plot(lossPlot)
    fig.canvas.draw()

print("\ndone")